Skip to content

feat: add provisioner job hang detector #7927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jun 25, 2023
Merged

feat: add provisioner job hang detector #7927

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
a38e949
feat: add provisioner job hang detector
deansheather Jun 9, 2023
dfaf836
Merge branch 'main' into dean/hang-detector
deansheather Jun 9, 2023
530e8f2
chore: ensure provisioner kills upon failed update
deansheather Jun 9, 2023
027443a
Merge branch 'main' into dean/hang-detector
deansheather Jun 13, 2023
0218151
fix cancel timeout test
deansheather Jun 13, 2023
9e0ae3b
fixup! fix cancel timeout test
deansheather Jun 13, 2023
590f76a
comments
deansheather Jun 20, 2023
6f1e127
Merge branch 'main' into dean/hang-detector
deansheather Jun 20, 2023
e284b47
small refactor
deansheather Jun 21, 2023
0b9e78a
tx per unhang job
deansheather Jun 21, 2023
8f16c3b
Merge branch 'main' into dean/hang-detector
deansheather Jun 21, 2023
f25938a
fixup! Merge branch 'main' into dean/hang-detector
deansheather Jun 21, 2023
aa36a0d
Merge branch 'main' into dean/hang-detector
deansheather Jun 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix cancel timeout test
  • Loading branch information
@deansheather
deansheather committed Jun 13, 2023
commit 0218151925bb844af41ca8061e3aac9958aa1e3d
23 changes: 21 additions & 2 deletions provisioner/terraform/provision_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,16 @@ func TestProvision_CancelTimeout(t *testing.T) {
t.Skip("This test uses interrupts and is not supported on Windows")
}

cwd, err := os.Getwd()
require.NoError(t, err)
fakeBin := filepath.Join(cwd, "testdata", "fake_cancel_hang.sh")

dir := t.TempDir()
binPath := filepath.Join(dir, "terraform")

content := "#!/bin/sh\nset -eu\nsleep 15"
err := os.WriteFile(binPath, []byte(content), 0o755) //#nosec
// Example: exec /path/to/terrafork_fake_cancel.sh 1.2.1 apply "$@"
content := fmt.Sprintf("#!/bin/sh\nexec %q %s \"$@\"\n", fakeBin, terraform.TerraformVersion.String())
err = os.WriteFile(binPath, []byte(content), 0o755) //#nosec
require.NoError(t, err)

ctx, api := setupProvisioner(t, &provisionerServeOptions{
Expand All @@ -218,6 +223,20 @@ func TestProvision_CancelTimeout(t *testing.T) {
})
require.NoError(t, err)

for _, line := range []string{"init", "apply_start"} {
LoopStart:
msg, err := response.Recv()
require.NoError(t, err)

t.Log(msg.Type)

log := msg.GetLog()
if log == nil {
goto LoopStart
}
require.Equal(t, line, log.Output)
}

err = response.Send(&proto.Provision_Request{
Type: &proto.Provision_Request_Cancel{
Cancel: &proto.Provision_Cancel{},
Expand Down
41 changes: 41 additions & 0 deletions provisioner/terraform/testdata/fake_cancel_hang.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#!/bin/sh

VERSION=$1
shift 1

json_print() {
echo "{\"@level\":\"error\",\"@message\":\"$*\"}"
}

case "$1" in
version)
cat <<-EOF
{
"terraform_version": "${VERSION}",
"platform": "linux_amd64",
"provider_selections": {},
"terraform_outdated": false
}
EOF
exit 0
;;
init)
echo "init"
exit 0
;;
apply)
trap 'json_print interrupt' INT

json_print apply_start
sleep 10 2>/dev/null >/dev/null
json_print apply_end

exit 0
;;
plan)
echo "plan not supported"
exit 1
;;
esac

exit 0