Merge branch '152-snapshot-customization' into 'master'

agneum · agneum · commit 090aaacc205e · 2020-09-08T15:24:28.000Z
fix: allow to use custom Docker images for data promotion, add preprocessing runner (#152) See merge request postgres-ai/database-lab!167
diff --git a/configs/config.example.physical_generic.yml b/configs/config.example.physical_generic.yml
@@ -153,6 +153,9 @@ retrieval:
         # Promote PGDATA after data fetching.
         promote: true
 
+        # Custom Docker image for data promotion.
+        dockerImage: "postgresai/sync-instance:12"
+
         # It is possible to define a pre-precessing script. For example, "/tmp/scripts/custom.sh".
         # Default: empty string (no pre-processing defined).
         # This can be used for scrubbing eliminating PII data, to define data masking, etc.
diff --git a/configs/config.example.physical_walg.yml b/configs/config.example.physical_walg.yml
@@ -142,6 +142,9 @@ retrieval:
         # Promote PGDATA after data fetching.
         promote: true
 
+        # Custom Docker image for data promotion.
+        dockerImage: "postgresai/sync-instance:12"
+
         # It is possible to define a pre-precessing script. For example, "/tmp/scripts/custom.sh".
         # Default: empty string (no pre-processing defined).
         # This can be used for scrubbing eliminating PII data, to define data masking, etc.
diff --git a/pkg/retrieval/engine/postgres/snapshot/physical.go b/pkg/retrieval/engine/postgres/snapshot/physical.go
@@ -70,6 +70,7 @@ type PhysicalInitial struct {
 // PhysicalOptions describes options for a physical initialization job.
 type PhysicalOptions struct {
 	Promote             bool              `yaml:"promote"`
+	DockerImage         string            `yaml:"dockerImage"`
 	PreprocessingScript string            `yaml:"preprocessingScript"`
 	Configs             map[string]string `yaml:"configs"`
 	Scheduler           *Scheduler        `yaml:"scheduler"`
@@ -329,7 +330,10 @@ func (p *PhysicalInitial) promoteInstance(ctx context.Context, clonePath string)
 		return errors.Wrap(err, "failed to build container host config")
 	}
 
-	promoteImage := fmt.Sprintf("postgresai/sync-instance:%s", pgVersion)
+	promoteImage := p.options.DockerImage
+	if promoteImage == "" {
+		promoteImage = fmt.Sprintf("postgresai/sync-instance:%s", pgVersion)
+	}
 
 	if err := tools.PullImage(ctx, p.dockerClient, promoteImage); err != nil {
 		return errors.Wrap(err, "failed to scan image pulling response")
diff --git a/pkg/retrieval/engine/postgres/snapshot/snapshot.go b/pkg/retrieval/engine/postgres/snapshot/snapshot.go
@@ -8,13 +8,13 @@ package snapshot
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"strings"
 
 	"github.com/pkg/errors"
 
 	"gitlab.com/postgres-ai/database-lab/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
+	"gitlab.com/postgres-ai/database-lab/pkg/services/provision/runners"
 )
 
 func extractDataStateAt(dbMarker *dbmarker.Marker) string {
@@ -31,12 +31,12 @@ func extractDataStateAt(dbMarker *dbmarker.Marker) string {
 }
 
 func runPreprocessingScript(preprocessingScript string) error {
-	commandOutput, err := exec.Command("bash", preprocessingScript).Output()
+	commandOutput, err := runners.NewLocalRunner(false).Run(preprocessingScript)
 	if err != nil {
 		return errors.Wrap(err, "failed to run custom script")
 	}
 
-	log.Msg(string(commandOutput))
+	log.Msg(commandOutput)
 
 	return nil
 }

Original file line number	Diff line number	Diff line change
`@@ -8,13 +8,13 @@ package snapshot`
`8`	`8`	`import (`
`9`	`9`	`"fmt"`
`10`	`10`	`"os"`
`11`		`- "os/exec"`
`12`	`11`	`"strings"`
`13`	`12`
`14`	`13`	`"github.com/pkg/errors"`
`15`	`14`
`16`	`15`	`"gitlab.com/postgres-ai/database-lab/pkg/log"`
`17`	`16`	`"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"`
	`17`	`+ "gitlab.com/postgres-ai/database-lab/pkg/services/provision/runners"`
`18`	`18`	`)`
`19`	`19`
`20`	`20`	`func extractDataStateAt(dbMarker *dbmarker.Marker) string {`
`@@ -31,12 +31,12 @@ func extractDataStateAt(dbMarker *dbmarker.Marker) string {`
`31`	`31`	`}`
`32`	`32`
`33`	`33`	`func runPreprocessingScript(preprocessingScript string) error {`
`34`		`- commandOutput, err := exec.Command("bash", preprocessingScript).Output()`
	`34`	`+ commandOutput, err := runners.NewLocalRunner(false).Run(preprocessingScript)`
`35`	`35`	`if err != nil {`
`36`	`36`	`return errors.Wrap(err, "failed to run custom script")`
`37`	`37`	`}`
`38`	`38`
`39`		`- log.Msg(string(commandOutput))`
	`39`	`+ log.Msg(commandOutput)`
`40`	`40`
`41`	`41`	`return nil`
`42`	`42`	`}`