Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commita400fb7

Browse files
committed
feat: automated data retrieval - physical and logical restoring (#93)
1 parent3fa22fc commita400fb7

File tree

30 files changed

+1445
-18
lines changed

30 files changed

+1445
-18
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@
88
/deploy/
99

1010
/configs/config.yml
11+
/configs/retrieval.yml

‎cmd/database-lab/main.go‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ import (
1414
"bytes"
1515
"context"
1616

17+
"github.com/docker/docker/client"
1718
"github.com/jessevdk/go-flags"
1819
"github.com/pkg/errors"
1920

2021
"gitlab.com/postgres-ai/database-lab/pkg/config"
2122
"gitlab.com/postgres-ai/database-lab/pkg/log"
23+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval"
2224
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
2325
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
2426
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
@@ -36,8 +38,6 @@ var opts struct {
3638
}
3739

3840
funcmain() {
39-
ctx:=context.Background()
40-
4141
// Load CLI options.
4242
if_,err:=parseArgs();err!=nil {
4343
ifflags.WroteHelp(err) {
@@ -68,24 +68,41 @@ func main() {
6868
cfg.Provision.ModeLocal.DockerImage=opts.DockerImage
6969
}
7070

71-
provisionSvc,err:=provision.NewProvision(ctx,cfg.Provision)
71+
dockerCLI,err:=client.NewClientWithOpts(client.FromEnv)
72+
iferr!=nil {
73+
log.Fatal("Failed to create a Docker client:",err)
74+
}
75+
76+
ctx,cancel:=context.WithCancel(context.Background())
77+
defercancel()
78+
79+
// Create a new retrieval service to prepare a data directory and start snapshotting.
80+
retrievalSvc,err:=retrieval.New(cfg,dockerCLI)
81+
iferr!=nil {
82+
log.Fatal("Failed to build a retrieval service:",err)
83+
}
84+
85+
iferr:=retrievalSvc.Run(ctx);err!=nil {
86+
log.Fatal("Failed to run the data retrieval service:",err)
87+
}
88+
89+
// Create a cloning service to provision new clones.
90+
provisionSvc,err:=provision.New(ctx,cfg.Provision)
7291
iferr!=nil {
7392
log.Fatalf(errors.WithMessage(err,`error in "provision" config`))
7493
}
7594

76-
cloningSvc,err:=cloning.NewCloning(&cfg.Cloning,provisionSvc)
95+
cloningSvc,err:=cloning.New(&cfg.Cloning,provisionSvc)
7796
iferr!=nil {
7897
log.Fatalf(errors.WithMessage(err,"failed to init a new cloning service"))
7998
}
8099

81-
ctx,cancel:=context.WithCancel(context.Background())
82-
defercancel()
83-
84100
iferr=cloningSvc.Run(ctx);err!=nil {
85101
log.Fatalf(err)
86102
}
87103

88-
platformSvc:=platform.NewService(cfg.Platform)
104+
// Create a platform service to verify Platform tokens.
105+
platformSvc:=platform.New(cfg.Platform)
89106
iferr:=platformSvc.Init(ctx);err!=nil {
90107
log.Fatalf(errors.WithMessage(err,"failed to create a new platform service"))
91108
}
@@ -94,6 +111,7 @@ func main() {
94111
cfg.Server.VerificationToken=opts.VerificationToken
95112
}
96113

114+
// Start the Database Lab.
97115
server:=srv.NewServer(&cfg.Server,cloningSvc,platformSvc)
98116
iferr=server.Run();err!=nil {
99117
log.Fatalf(err)

‎configs/config.sample.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ server:
1111
# HTTP server port. By default: 2345.
1212
port:2345
1313

14+
global:
15+
engine:postgres
16+
dataDir:/var/lib/dblab
17+
dockerImage:"postgres:12-alpine"
18+
1419
# Postgres.ai Platform integration.
1520
platform:
1621
# API URL.
@@ -82,4 +87,30 @@ cloning:
8287
# 0 - disable automatic deletion.
8388
maxIdleMinutes:120
8489

90+
retrieval:
91+
stages:
92+
-initialize
93+
94+
spec:
95+
initialize:
96+
jobs:
97+
# - name: logical-restore
98+
# options:
99+
# dumpFile: /tmp/db.dump
100+
# forceInit: false
101+
# dbName: test
102+
# partial:
103+
# tables:
104+
# - test
105+
-name:physical-restore
106+
options:
107+
tool:walg
108+
dockerImage:"postgresai/sync-instance:12"
109+
envs:
110+
WALG_GS_PREFIX:"gs://{BUCKET}/{SCOPE}"
111+
walg:
112+
storage:gcs
113+
backupName:LATEST
114+
credentialsFile:/tmp/sa.json# optional
115+
85116
debug:true

‎go.mod‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,20 @@ go 1.13
44

55
require (
66
github.com/AlekSi/pointerv1.1.0
7+
github.com/Azure/go-ansitermv0.0.0-20170929234023-d6e3b3328b78// indirect
78
github.com/Microsoft/go-winiov0.4.14// indirect
9+
github.com/containerd/containerdv1.3.4// indirect
810
github.com/docker/distributionv2.7.1+incompatible// indirect
911
github.com/docker/dockerv1.13.1
1012
github.com/docker/go-connectionsv0.4.0// indirect
1113
github.com/docker/go-unitsv0.4.0// indirect
14+
github.com/gogo/protobufv1.3.1// indirect
1215
github.com/gorilla/muxv1.7.3
1316
github.com/jessevdk/go-flagsv1.4.1-0.20181221193153-c0795c8afcf4
1417
github.com/lib/pqv1.3.0
18+
github.com/morikuni/aecv1.0.0// indirect
1519
github.com/opencontainers/go-digestv1.0.0-rc1// indirect
20+
github.com/opencontainers/image-specv1.0.1// indirect
1621
github.com/pkg/errorsv0.8.1
1722
github.com/rs/xidv1.2.1
1823
github.com/sergi/go-diffv1.1.0
@@ -22,5 +27,9 @@ require (
2227
golang.org/x/cryptov0.0.0-20200604202706-70a84ac30bf9
2328
golang.org/x/netv0.0.0-20200114155413-6afb5195e5aa// indirect
2429
golang.org/x/sysv0.0.0-20200615200032-f1bc736245b1// indirect
30+
golang.org/x/timev0.0.0-20200416051211-89c76fbcd5d1// indirect
31+
google.golang.org/grpcv1.30.0// indirect
2532
gopkg.in/yaml.v2v2.2.7
2633
)
34+
35+
replacegithub.com/docker/dockerv1.13.1 =>github.com/docker/enginev17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible

‎go.sum‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,50 @@
1+
cloud.google.com/gov0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
12
github.com/AlekSi/pointerv1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI=
23
github.com/AlekSi/pointerv1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE=
4+
github.com/Azure/go-ansitermv0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
5+
github.com/Azure/go-ansitermv0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
36
github.com/BurntSushi/tomlv0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
47
github.com/BurntSushi/tomlv0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
58
github.com/Microsoft/go-winiov0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
69
github.com/Microsoft/go-winiov0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
10+
github.com/census-instrumentation/opencensus-protov0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
11+
github.com/client9/misspellv0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
12+
github.com/cncf/udpa/gov0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
13+
github.com/containerd/containerdv1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI=
14+
github.com/containerd/containerdv1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
715
github.com/cpuguy83/go-md2man/v2v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
816
github.com/cpuguy83/go-md2man/v2v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
917
github.com/davecgh/go-spewv1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1018
github.com/davecgh/go-spewv1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
1119
github.com/davecgh/go-spewv1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1220
github.com/docker/distributionv2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
1321
github.com/docker/distributionv2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
14-
github.com/docker/dockerv1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
15-
github.com/docker/dockerv1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
22+
github.com/docker/enginev17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible h1:5uC2w1yhWUPlduFVWY/W3wiluc85Cv8nxIKWPSFk/WY=
23+
github.com/docker/enginev17.12.0-ce-rc1.0.20200531234253-77e06fda0c94+incompatible/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
1624
github.com/docker/go-connectionsv0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
1725
github.com/docker/go-connectionsv0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
1826
github.com/docker/go-unitsv0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
1927
github.com/docker/go-unitsv0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
28+
github.com/envoyproxy/go-control-planev0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
29+
github.com/envoyproxy/go-control-planev0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
30+
github.com/envoyproxy/protoc-gen-validatev0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
31+
github.com/gogo/protobufv1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
32+
github.com/gogo/protobufv1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
33+
github.com/golang/glogv0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
34+
github.com/golang/mockv1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
35+
github.com/golang/protobufv1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
36+
github.com/golang/protobufv1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
37+
github.com/golang/protobufv1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
38+
github.com/golang/protobufv1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
39+
github.com/google/go-cmpv0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
40+
github.com/google/go-cmpv0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
41+
github.com/google/go-cmpv0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
2042
github.com/gorilla/muxv1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
2143
github.com/gorilla/muxv1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
2244
github.com/jessevdk/go-flagsv1.4.1-0.20181221193153-c0795c8afcf4 h1:xKkUL6QBojwguhKKetf1SocCAKqc6W7S/mGm9xEGllo=
2345
github.com/jessevdk/go-flagsv1.4.1-0.20181221193153-c0795c8afcf4/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
46+
github.com/kisielk/errcheckv1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
47+
github.com/kisielk/gotoolv1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
2448
github.com/konsorten/go-windows-terminal-sequencesv1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
2549
github.com/konsorten/go-windows-terminal-sequencesv1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
2650
github.com/kr/prettyv0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -30,12 +54,17 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
3054
github.com/kr/textv0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
3155
github.com/lib/pqv1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
3256
github.com/lib/pqv1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
57+
github.com/morikuni/aecv1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
58+
github.com/morikuni/aecv1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
3359
github.com/opencontainers/go-digestv1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
3460
github.com/opencontainers/go-digestv1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
61+
github.com/opencontainers/image-specv1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
62+
github.com/opencontainers/image-specv1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
3563
github.com/pkg/errorsv0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
3664
github.com/pkg/errorsv0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
3765
github.com/pmezard/go-difflibv1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
3866
github.com/pmezard/go-difflibv1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
67+
github.com/prometheus/client_modelv0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
3968
github.com/rs/xidv1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
4069
github.com/rs/xidv1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
4170
github.com/russross/blackfriday/v2v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -57,9 +86,22 @@ github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2
5786
golang.org/x/cryptov0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
5887
golang.org/x/cryptov0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM=
5988
golang.org/x/cryptov0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
89+
golang.org/x/expv0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
90+
golang.org/x/lintv0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
91+
golang.org/x/lintv0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
92+
golang.org/x/lintv0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
93+
golang.org/x/netv0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
94+
golang.org/x/netv0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
95+
golang.org/x/netv0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
96+
golang.org/x/netv0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
6097
golang.org/x/netv0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
6198
golang.org/x/netv0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
6299
golang.org/x/netv0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
100+
golang.org/x/oauth2v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
101+
golang.org/x/syncv0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
102+
golang.org/x/syncv0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
103+
golang.org/x/syncv0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
104+
golang.org/x/sysv0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
63105
golang.org/x/sysv0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
64106
golang.org/x/sysv0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
65107
golang.org/x/sysv0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -69,11 +111,35 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qd
69111
golang.org/x/sysv0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
70112
golang.org/x/sysv0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
71113
golang.org/x/sysv0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
114+
golang.org/x/textv0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
72115
golang.org/x/textv0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
116+
golang.org/x/timev0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
117+
golang.org/x/timev0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
118+
golang.org/x/toolsv0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
119+
golang.org/x/toolsv0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
120+
golang.org/x/toolsv0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
121+
golang.org/x/toolsv0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
122+
golang.org/x/toolsv0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
123+
golang.org/x/xerrorsv0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
124+
golang.org/x/xerrorsv0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
125+
google.golang.org/appenginev1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
126+
google.golang.org/appenginev1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
127+
google.golang.org/genprotov0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
128+
google.golang.org/genprotov0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
129+
google.golang.org/genprotov0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
130+
google.golang.org/grpcv1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
131+
google.golang.org/grpcv1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
132+
google.golang.org/grpcv1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
133+
google.golang.org/grpcv1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE=
134+
google.golang.org/grpcv1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
73135
gopkg.in/check.v1v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
74136
gopkg.in/check.v1v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
75137
gopkg.in/check.v1v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
76138
gopkg.in/yaml.v2v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
77139
gopkg.in/yaml.v2v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
78140
gopkg.in/yaml.v2v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
79141
gopkg.in/yaml.v2v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
142+
gotest.toolsv2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
143+
gotest.toolsv2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
144+
honnef.co/go/toolsv0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
145+
honnef.co/go/toolsv0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

‎pkg/config/config.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"io/ioutil"
1111
"os/user"
1212

13+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
1314
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
1415
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
1516
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
@@ -26,9 +27,18 @@ type Config struct {
2627
Provision provision.Config`yaml:"provision"`
2728
Cloning cloning.Config`yaml:"cloning"`
2829
Platform platform.Config`yaml:"platform"`
30+
GlobalGlobal`yaml:"global"`
31+
Retrieval config.Config`yaml:"retrieval"`
2932
Debugbool`yaml:"debug"`
3033
}
3134

35+
// Global contains global Database Lab configurations.
36+
typeGlobalstruct {
37+
Enginestring`yaml:"engine"`
38+
DataDirstring`yaml:"dataDir"`
39+
DockerImagestring`yaml:"dockerImage"`
40+
}
41+
3242
// LoadConfig instances a new Config by configuration filename.
3343
funcLoadConfig(namestring) (*Config,error) {
3444
path,err:=util.GetConfigPath(name)
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
2020 © Postgres.ai
3+
*/
4+
5+
// Package components provides the key components of data retrieval.
6+
package components
7+
8+
import (
9+
"context"
10+
11+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
12+
)
13+
14+
// StageBuilder builds a StageRunner.
15+
typeStageBuilderinterface {
16+
BuildStageRunner(string) (StageRunner,error)
17+
}
18+
19+
// JobRunner performs a job.
20+
typeJobRunnerinterface {
21+
// Name returns a job name.
22+
Name()string
23+
24+
// Run starts a job.
25+
Run(ctx context.Context)error
26+
}
27+
28+
// StageRunner declares stage content and performs stage jobs.
29+
typeStageRunnerinterface {
30+
// AddJob applies jobs to the current stage.
31+
AddJob(JobRunner)
32+
33+
// BuildJob builds stage jobs.
34+
BuildJob(config.JobConfig) (JobRunner,error)
35+
36+
// Run starts stage jobs.
37+
Run(ctx context.Context)error
38+
}

‎pkg/retrieval/config/config.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
2020 © Postgres.ai
3+
*/
4+
5+
// Package config contains configuration options of the data retrieval.
6+
package config
7+
8+
// Config describes of data retrieval stages.
9+
typeConfigstruct {
10+
Stages []string`yaml:"stages,flow"`
11+
StageSpecmap[string]Spec`yaml:"spec"`
12+
}
13+
14+
// Spec describes a retrieval job.
15+
typeSpecstruct {
16+
Jobs []JobConfig`yaml:"jobs,flow"`
17+
}
18+
19+
// JobConfig contains details about a job.
20+
typeJobConfigstruct {
21+
Namestring`yaml:"name"`
22+
Optionsmap[string]interface{}`yaml:"options"`
23+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp