Skip to content

fix: Allow terraform provisions to be gracefully cancelled #3526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Aug 18, 2022
Merged

fix: Allow terraform provisions to be gracefully cancelled #3526

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
fd47d05
fix: Allow terraform provisions to be gracefully cancelled
mafredri Aug 16, 2022
72f723e
Cleanup
mafredri Aug 16, 2022
00c7746
Revert protoc version bumps
mafredri Aug 16, 2022
7df45e1
fix: Remove unused import from Windows
mafredri Aug 16, 2022
211b7f9
fix: Add build tags
mafredri Aug 16, 2022
3121f6a
fix: Amend PR comments
mafredri Aug 16, 2022
27fe6f0
chore: Revert proto changes
mafredri Aug 16, 2022
1d7f825
chore: Revert newline in proto
mafredri Aug 16, 2022
5cbb712
fix: Remove incorrect minimumTerraformVersion variable
mafredri Aug 17, 2022
3e4005a
fix: Add ExitTimeout to terraform.ServeOptions
mafredri Aug 17, 2022
685c583
fix: Add test for interrupt/cancellation
mafredri Aug 17, 2022
3303ee1
Allow init to return provision complete response
mafredri Aug 17, 2022
db41a65
Add testdata script
mafredri Aug 17, 2022
80a4912
Remove superfluous require
mafredri Aug 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 85 additions & 34 deletions provisioner/terraform/executor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (e executor) basicEnv() []string {
return env
}

func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOutWriter, stdErrWriter io.WriteCloser) (err error) {
func (e executor) execWriteOutput(ctx, killCtx context.Context, args, env []string, stdOutWriter, stdErrWriter io.WriteCloser) (err error) {
defer func() {
closeErr := stdOutWriter.Close()
if err == nil && closeErr != nil {
Expand All @@ -52,8 +52,12 @@ func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOu
err = closeErr
}
}()
if ctx.Err() != nil {
return ctx.Err()
}

// #nosec
cmd := exec.CommandContext(ctx, e.binaryPath, args...)
cmd := exec.CommandContext(killCtx, e.binaryPath, args...)
cmd.Dir = e.workdir
cmd.Env = env

Expand All @@ -63,19 +67,36 @@ func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOu
cmd.Stdout = syncWriter{mut, stdOutWriter}
cmd.Stderr = syncWriter{mut, stdErrWriter}

return cmd.Run()
err = cmd.Start()
if err != nil {
return err
}
interruptCommandOnCancel(ctx, killCtx, cmd)

return cmd.Wait()
}

func (e executor) execParseJSON(ctx context.Context, args, env []string, v interface{}) error {
func (e executor) execParseJSON(ctx, killCtx context.Context, args, env []string, v interface{}) error {
if ctx.Err() != nil {
return ctx.Err()
}

// #nosec
cmd := exec.CommandContext(ctx, e.binaryPath, args...)
cmd := exec.CommandContext(killCtx, e.binaryPath, args...)
cmd.Dir = e.workdir
cmd.Env = env
out := &bytes.Buffer{}
stdErr := &bytes.Buffer{}
cmd.Stdout = out
cmd.Stderr = stdErr
err := cmd.Run()

err := cmd.Start()
if err != nil {
return err
}
interruptCommandOnCancel(ctx, killCtx, cmd)

err = cmd.Wait()
if err != nil {
errString, _ := io.ReadAll(stdErr)
return xerrors.Errorf("%s: %w", errString, err)
Expand All @@ -95,11 +116,11 @@ func (e executor) checkMinVersion(ctx context.Context) error {
if err != nil {
return err
}
if !v.GreaterThanOrEqual(minimumTerraformVersion) {
if !v.GreaterThanOrEqual(minTerraformVersion) {
return xerrors.Errorf(
"terraform version %q is too old. required >= %q",
v.String(),
minimumTerraformVersion.String())
minTerraformVersion.String())
}
return nil
}
Expand All @@ -109,6 +130,10 @@ func (e executor) version(ctx context.Context) (*version.Version, error) {
}

func versionFromBinaryPath(ctx context.Context, binaryPath string) (*version.Version, error) {
if ctx.Err() != nil {
return nil, ctx.Err()
}

// #nosec
cmd := exec.CommandContext(ctx, binaryPath, "version", "-json")
out, err := cmd.Output()
Expand All @@ -130,7 +155,7 @@ func versionFromBinaryPath(ctx context.Context, binaryPath string) (*version.Ver
return version.NewVersion(vj.Version)
}

func (e executor) init(ctx context.Context, logr logger) error {
func (e executor) init(ctx, killCtx context.Context, logr logger) error {
outWriter, doneOut := logWriter(logr, proto.LogLevel_DEBUG)
errWriter, doneErr := logWriter(logr, proto.LogLevel_ERROR)
defer func() {
Expand All @@ -156,11 +181,11 @@ func (e executor) init(ctx context.Context, logr logger) error {
defer e.initMu.Unlock()
}

return e.execWriteOutput(ctx, args, e.basicEnv(), outWriter, errWriter)
return e.execWriteOutput(ctx, killCtx, args, e.basicEnv(), outWriter, errWriter)
}

// revive:disable-next-line:flag-parameter
func (e executor) plan(ctx context.Context, env, vars []string, logr logger, destroy bool) (*proto.Provision_Response, error) {
func (e executor) plan(ctx, killCtx context.Context, env, vars []string, logr logger, destroy bool) (*proto.Provision_Response, error) {
planfilePath := filepath.Join(e.workdir, "terraform.tfplan")
args := []string{
"plan",
Expand All @@ -184,11 +209,11 @@ func (e executor) plan(ctx context.Context, env, vars []string, logr logger, des
<-doneErr
}()

err := e.execWriteOutput(ctx, args, env, outWriter, errWriter)
err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter)
if err != nil {
return nil, xerrors.Errorf("terraform plan: %w", err)
}
resources, err := e.planResources(ctx, planfilePath)
resources, err := e.planResources(ctx, killCtx, planfilePath)
if err != nil {
return nil, err
}
Expand All @@ -201,40 +226,52 @@ func (e executor) plan(ctx context.Context, env, vars []string, logr logger, des
}, nil
}

func (e executor) planResources(ctx context.Context, planfilePath string) ([]*proto.Resource, error) {
plan, err := e.showPlan(ctx, planfilePath)
func (e executor) planResources(ctx, killCtx context.Context, planfilePath string) ([]*proto.Resource, error) {
plan, err := e.showPlan(ctx, killCtx, planfilePath)
if err != nil {
return nil, xerrors.Errorf("show terraform plan file: %w", err)
}

rawGraph, err := e.graph(ctx)
rawGraph, err := e.graph(ctx, killCtx)
if err != nil {
return nil, xerrors.Errorf("graph: %w", err)
}
return ConvertResources(plan.PlannedValues.RootModule, rawGraph)
}

func (e executor) showPlan(ctx context.Context, planfilePath string) (*tfjson.Plan, error) {
func (e executor) showPlan(ctx, killCtx context.Context, planfilePath string) (*tfjson.Plan, error) {
args := []string{"show", "-json", "-no-color", planfilePath}
p := new(tfjson.Plan)
err := e.execParseJSON(ctx, args, e.basicEnv(), p)
err := e.execParseJSON(ctx, killCtx, args, e.basicEnv(), p)
return p, err
}

func (e executor) graph(ctx context.Context) (string, error) {
// #nosec
cmd := exec.CommandContext(ctx, e.binaryPath, "graph")
func (e executor) graph(ctx, killCtx context.Context) (string, error) {
if ctx.Err() != nil {
return "", ctx.Err()
}

var out bytes.Buffer
cmd := exec.CommandContext(killCtx, e.binaryPath, "graph") // #nosec
cmd.Stdout = &out
cmd.Dir = e.workdir
cmd.Env = e.basicEnv()
out, err := cmd.Output()

err := cmd.Start()
if err != nil {
return "", err
}
interruptCommandOnCancel(ctx, killCtx, cmd)

err = cmd.Wait()
if err != nil {
return "", xerrors.Errorf("graph: %w", err)
}
return string(out), nil
return out.String(), nil
}

// revive:disable-next-line:flag-parameter
func (e executor) apply(ctx context.Context, env, vars []string, logr logger, destroy bool,
func (e executor) apply(ctx, killCtx context.Context, env, vars []string, logr logger, destroy bool,
) (*proto.Provision_Response, error) {
args := []string{
"apply",
Expand All @@ -258,11 +295,11 @@ func (e executor) apply(ctx context.Context, env, vars []string, logr logger, de
<-doneErr
}()

err := e.execWriteOutput(ctx, args, env, outWriter, errWriter)
err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter)
if err != nil {
return nil, xerrors.Errorf("terraform apply: %w", err)
}
resources, err := e.stateResources(ctx)
resources, err := e.stateResources(ctx, killCtx)
if err != nil {
return nil, err
}
Expand All @@ -281,12 +318,12 @@ func (e executor) apply(ctx context.Context, env, vars []string, logr logger, de
}, nil
}

func (e executor) stateResources(ctx context.Context) ([]*proto.Resource, error) {
state, err := e.state(ctx)
func (e executor) stateResources(ctx, killCtx context.Context) ([]*proto.Resource, error) {
state, err := e.state(ctx, killCtx)
if err != nil {
return nil, err
}
rawGraph, err := e.graph(ctx)
rawGraph, err := e.graph(ctx, killCtx)
if err != nil {
return nil, xerrors.Errorf("get terraform graph: %w", err)
}
Expand All @@ -300,16 +337,33 @@ func (e executor) stateResources(ctx context.Context) ([]*proto.Resource, error)
return resources, nil
}

func (e executor) state(ctx context.Context) (*tfjson.State, error) {
func (e executor) state(ctx, killCtx context.Context) (*tfjson.State, error) {
args := []string{"show", "-json"}
state := &tfjson.State{}
err := e.execParseJSON(ctx, args, e.basicEnv(), state)
err := e.execParseJSON(ctx, killCtx, args, e.basicEnv(), state)
if err != nil {
return nil, xerrors.Errorf("terraform show state: %w", err)
}
return state, nil
}

func interruptCommandOnCancel(ctx, killCtx context.Context, cmd *exec.Cmd) {
go func() {
select {
case <-ctx.Done():
switch runtime.GOOS {
case "windows":
// Interrupts aren't supported by Windows.
_ = cmd.Process.Kill()
default:
_ = cmd.Process.Signal(os.Interrupt)
}

case <-killCtx.Done():
}
}()
}

type logger interface {
Log(*proto.Log) error
}
Expand Down Expand Up @@ -381,9 +435,6 @@ func provisionReadAndLog(logr logger, reader io.Reader, done chan<- any) {

// If the diagnostic is provided, let's provide a bit more info!
logLevel = convertTerraformLogLevel(log.Diagnostic.Severity, logr)
if err != nil {
continue
}
err = logr.Log(&proto.Log{Level: logLevel, Output: log.Diagnostic.Detail})
if err != nil {
// Not much we can do. We can't log because logging is itself breaking!
Expand Down
4 changes: 2 additions & 2 deletions provisioner/terraform/parse_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
func TestParse(t *testing.T) {
t.Parallel()

ctx, api := setupProvisioner(t)
ctx, api := setupProvisioner(t, nil)

testCases := []struct {
Name string
Expand Down Expand Up @@ -171,7 +171,7 @@ func TestParse(t *testing.T) {
// Write all files to the temporary test directory.
directory := t.TempDir()
for path, content := range testCase.Files {
err := os.WriteFile(filepath.Join(directory, path), []byte(content), 0600)
err := os.WriteFile(filepath.Join(directory, path), []byte(content), 0o600)
require.NoError(t, err)
}

Expand Down
Loading