Skip to content

Commit f142345

Browse files
mafredrimafredri
authored
fix: Allow terraform provisions to be gracefully cancelled (#3526)
* fix: Allow terraform provisions to be gracefully cancelled This change allows terraform commands to be gracefully cancelled on Unix-like platforms by signaling interrupt on provision cancellation. One implementation detail to note is that we do not necessarily kill a running terraform command immediately even if the stream is closed. The reason for this is to allow for graceful cancellation even in such an event. Currently the timeout is set to 5 minutes by default. Related: #2683 The above issue may be partially or fully fixed by this change. * fix: Remove incorrect minimumTerraformVersion variable * Allow init to return provision complete response
1 parent 6a0f8ae commit f142345

File tree

6 files changed

+357
-84
lines changed

6 files changed

+357
-84
lines changed

‎provisioner/terraform/executor.go

+85-34
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (e executor) basicEnv() []string {
4141
return env
4242
}
4343

44-
func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOutWriter, stdErrWriter io.WriteCloser) (err error) {
44+
func (e executor) execWriteOutput(ctx, killCtx context.Context, args, env []string, stdOutWriter, stdErrWriter io.WriteCloser) (err error) {
4545
defer func() {
4646
closeErr := stdOutWriter.Close()
4747
if err == nil && closeErr != nil {
@@ -52,8 +52,12 @@ func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOu
5252
err = closeErr
5353
}
5454
}()
55+
if ctx.Err() != nil {
56+
return ctx.Err()
57+
}
58+
5559
// #nosec
56-
cmd := exec.CommandContext(ctx, e.binaryPath, args...)
60+
cmd := exec.CommandContext(killCtx, e.binaryPath, args...)
5761
cmd.Dir = e.workdir
5862
cmd.Env = env
5963

@@ -63,19 +67,36 @@ func (e executor) execWriteOutput(ctx context.Context, args, env []string, stdOu
6367
cmd.Stdout = syncWriter{mut, stdOutWriter}
6468
cmd.Stderr = syncWriter{mut, stdErrWriter}
6569

66-
return cmd.Run()
70+
err = cmd.Start()
71+
if err != nil {
72+
return err
73+
}
74+
interruptCommandOnCancel(ctx, killCtx, cmd)
75+
76+
return cmd.Wait()
6777
}
6878

69-
func (e executor) execParseJSON(ctx context.Context, args, env []string, v interface{}) error {
79+
func (e executor) execParseJSON(ctx, killCtx context.Context, args, env []string, v interface{}) error {
80+
if ctx.Err() != nil {
81+
return ctx.Err()
82+
}
83+
7084
// #nosec
71-
cmd := exec.CommandContext(ctx, e.binaryPath, args...)
85+
cmd := exec.CommandContext(killCtx, e.binaryPath, args...)
7286
cmd.Dir = e.workdir
7387
cmd.Env = env
7488
out := &bytes.Buffer{}
7589
stdErr := &bytes.Buffer{}
7690
cmd.Stdout = out
7791
cmd.Stderr = stdErr
78-
err := cmd.Run()
92+
93+
err := cmd.Start()
94+
if err != nil {
95+
return err
96+
}
97+
interruptCommandOnCancel(ctx, killCtx, cmd)
98+
99+
err = cmd.Wait()
79100
if err != nil {
80101
errString, _ := io.ReadAll(stdErr)
81102
return xerrors.Errorf("%s: %w", errString, err)
@@ -95,11 +116,11 @@ func (e executor) checkMinVersion(ctx context.Context) error {
95116
if err != nil {
96117
return err
97118
}
98-
if !v.GreaterThanOrEqual(minimumTerraformVersion) {
119+
if !v.GreaterThanOrEqual(minTerraformVersion) {
99120
return xerrors.Errorf(
100121
"terraform version %q is too old. required >= %q",
101122
v.String(),
102-
minimumTerraformVersion.String())
123+
minTerraformVersion.String())
103124
}
104125
return nil
105126
}
@@ -109,6 +130,10 @@ func (e executor) version(ctx context.Context) (*version.Version, error) {
109130
}
110131

111132
func versionFromBinaryPath(ctx context.Context, binaryPath string) (*version.Version, error) {
133+
if ctx.Err() != nil {
134+
return nil, ctx.Err()
135+
}
136+
112137
// #nosec
113138
cmd := exec.CommandContext(ctx, binaryPath, "version", "-json")
114139
out, err := cmd.Output()
@@ -130,7 +155,7 @@ func versionFromBinaryPath(ctx context.Context, binaryPath string) (*version.Ver
130155
return version.NewVersion(vj.Version)
131156
}
132157

133-
func (e executor) init(ctx context.Context, logr logger) error {
158+
func (e executor) init(ctx, killCtx context.Context, logr logger) error {
134159
outWriter, doneOut := logWriter(logr, proto.LogLevel_DEBUG)
135160
errWriter, doneErr := logWriter(logr, proto.LogLevel_ERROR)
136161
defer func() {
@@ -156,11 +181,11 @@ func (e executor) init(ctx context.Context, logr logger) error {
156181
defer e.initMu.Unlock()
157182
}
158183

159-
return e.execWriteOutput(ctx, args, e.basicEnv(), outWriter, errWriter)
184+
return e.execWriteOutput(ctx, killCtx, args, e.basicEnv(), outWriter, errWriter)
160185
}
161186

162187
// revive:disable-next-line:flag-parameter
163-
func (e executor) plan(ctx context.Context, env, vars []string, logr logger, destroy bool) (*proto.Provision_Response, error) {
188+
func (e executor) plan(ctx, killCtx context.Context, env, vars []string, logr logger, destroy bool) (*proto.Provision_Response, error) {
164189
planfilePath := filepath.Join(e.workdir, "terraform.tfplan")
165190
args := []string{
166191
"plan",
@@ -184,11 +209,11 @@ func (e executor) plan(ctx context.Context, env, vars []string, logr logger, des
184209
<-doneErr
185210
}()
186211

187-
err := e.execWriteOutput(ctx, args, env, outWriter, errWriter)
212+
err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter)
188213
if err != nil {
189214
return nil, xerrors.Errorf("terraform plan: %w", err)
190215
}
191-
resources, err := e.planResources(ctx, planfilePath)
216+
resources, err := e.planResources(ctx, killCtx, planfilePath)
192217
if err != nil {
193218
return nil, err
194219
}
@@ -201,40 +226,52 @@ func (e executor) plan(ctx context.Context, env, vars []string, logr logger, des
201226
}, nil
202227
}
203228

204-
func (e executor) planResources(ctx context.Context, planfilePath string) ([]*proto.Resource, error) {
205-
plan, err := e.showPlan(ctx, planfilePath)
229+
func (e executor) planResources(ctx, killCtx context.Context, planfilePath string) ([]*proto.Resource, error) {
230+
plan, err := e.showPlan(ctx, killCtx, planfilePath)
206231
if err != nil {
207232
return nil, xerrors.Errorf("show terraform plan file: %w", err)
208233
}
209234

210-
rawGraph, err := e.graph(ctx)
235+
rawGraph, err := e.graph(ctx, killCtx)
211236
if err != nil {
212237
return nil, xerrors.Errorf("graph: %w", err)
213238
}
214239
return ConvertResources(plan.PlannedValues.RootModule, rawGraph)
215240
}
216241

217-
func (e executor) showPlan(ctx context.Context, planfilePath string) (*tfjson.Plan, error) {
242+
func (e executor) showPlan(ctx, killCtx context.Context, planfilePath string) (*tfjson.Plan, error) {
218243
args := []string{"show", "-json", "-no-color", planfilePath}
219244
p := new(tfjson.Plan)
220-
err := e.execParseJSON(ctx, args, e.basicEnv(), p)
245+
err := e.execParseJSON(ctx, killCtx, args, e.basicEnv(), p)
221246
return p, err
222247
}
223248

224-
func (e executor) graph(ctx context.Context) (string, error) {
225-
// #nosec
226-
cmd := exec.CommandContext(ctx, e.binaryPath, "graph")
249+
func (e executor) graph(ctx, killCtx context.Context) (string, error) {
250+
if ctx.Err() != nil {
251+
return "", ctx.Err()
252+
}
253+
254+
var out bytes.Buffer
255+
cmd := exec.CommandContext(killCtx, e.binaryPath, "graph") // #nosec
256+
cmd.Stdout = &out
227257
cmd.Dir = e.workdir
228258
cmd.Env = e.basicEnv()
229-
out, err := cmd.Output()
259+
260+
err := cmd.Start()
261+
if err != nil {
262+
return "", err
263+
}
264+
interruptCommandOnCancel(ctx, killCtx, cmd)
265+
266+
err = cmd.Wait()
230267
if err != nil {
231268
return "", xerrors.Errorf("graph: %w", err)
232269
}
233-
return string(out), nil
270+
return out.String(), nil
234271
}
235272

236273
// revive:disable-next-line:flag-parameter
237-
func (e executor) apply(ctx context.Context, env, vars []string, logr logger, destroy bool,
274+
func (e executor) apply(ctx, killCtx context.Context, env, vars []string, logr logger, destroy bool,
238275
) (*proto.Provision_Response, error) {
239276
args := []string{
240277
"apply",
@@ -258,11 +295,11 @@ func (e executor) apply(ctx context.Context, env, vars []string, logr logger, de
258295
<-doneErr
259296
}()
260297

261-
err := e.execWriteOutput(ctx, args, env, outWriter, errWriter)
298+
err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter)
262299
if err != nil {
263300
return nil, xerrors.Errorf("terraform apply: %w", err)
264301
}
265-
resources, err := e.stateResources(ctx)
302+
resources, err := e.stateResources(ctx, killCtx)
266303
if err != nil {
267304
return nil, err
268305
}
@@ -281,12 +318,12 @@ func (e executor) apply(ctx context.Context, env, vars []string, logr logger, de
281318
}, nil
282319
}
283320

284-
func (e executor) stateResources(ctx context.Context) ([]*proto.Resource, error) {
285-
state, err := e.state(ctx)
321+
func (e executor) stateResources(ctx, killCtx context.Context) ([]*proto.Resource, error) {
322+
state, err := e.state(ctx, killCtx)
286323
if err != nil {
287324
return nil, err
288325
}
289-
rawGraph, err := e.graph(ctx)
326+
rawGraph, err := e.graph(ctx, killCtx)
290327
if err != nil {
291328
return nil, xerrors.Errorf("get terraform graph: %w", err)
292329
}
@@ -300,16 +337,33 @@ func (e executor) stateResources(ctx context.Context) ([]*proto.Resource, error)
300337
return resources, nil
301338
}
302339

303-
func (e executor) state(ctx context.Context) (*tfjson.State, error) {
340+
func (e executor) state(ctx, killCtx context.Context) (*tfjson.State, error) {
304341
args := []string{"show", "-json"}
305342
state := &tfjson.State{}
306-
err := e.execParseJSON(ctx, args, e.basicEnv(), state)
343+
err := e.execParseJSON(ctx, killCtx, args, e.basicEnv(), state)
307344
if err != nil {
308345
return nil, xerrors.Errorf("terraform show state: %w", err)
309346
}
310347
return state, nil
311348
}
312349

350+
func interruptCommandOnCancel(ctx, killCtx context.Context, cmd *exec.Cmd) {
351+
go func() {
352+
select {
353+
case <-ctx.Done():
354+
switch runtime.GOOS {
355+
case "windows":
356+
// Interrupts aren't supported by Windows.
357+
_ = cmd.Process.Kill()
358+
default:
359+
_ = cmd.Process.Signal(os.Interrupt)
360+
}
361+
362+
case <-killCtx.Done():
363+
}
364+
}()
365+
}
366+
313367
type logger interface {
314368
Log(*proto.Log) error
315369
}
@@ -381,9 +435,6 @@ func provisionReadAndLog(logr logger, reader io.Reader, done chan<- any) {
381435

382436
// If the diagnostic is provided, let's provide a bit more info!
383437
logLevel = convertTerraformLogLevel(log.Diagnostic.Severity, logr)
384-
if err != nil {
385-
continue
386-
}
387438
err = logr.Log(&proto.Log{Level: logLevel, Output: log.Diagnostic.Detail})
388439
if err != nil {
389440
// Not much we can do. We can't log because logging is itself breaking!

‎provisioner/terraform/parse_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
func TestParse(t *testing.T) {
1717
t.Parallel()
1818

19-
ctx, api := setupProvisioner(t)
19+
ctx, api := setupProvisioner(t, nil)
2020

2121
testCases := []struct {
2222
Name string
@@ -171,7 +171,7 @@ func TestParse(t *testing.T) {
171171
// Write all files to the temporary test directory.
172172
directory := t.TempDir()
173173
for path, content := range testCase.Files {
174-
err := os.WriteFile(filepath.Join(directory, path), []byte(content), 0600)
174+
err := os.WriteFile(filepath.Join(directory, path), []byte(content), 0o600)
175175
require.NoError(t, err)
176176
}
177177

0 commit comments

Comments
 (0)