Skip to content

feat: in-process provisionerd connection #1568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 19, 2022
Merged

feat: in-process provisionerd connection #1568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
09b582d
in-process provisionerd connection
spikecurtis May 17, 2022
82d52a0
disable lint for server.go/newProvisionerDaemon
spikecurtis May 18, 2022
1f1d7f9
Merge branch 'main' of github.com:coder/coder into spike/1375_in_proc…
spikecurtis May 19, 2022
7b2e5df
Merge branch 'main' of github.com:coder/coder into spike/1375_in_proc…
spikecurtis May 19, 2022
7240300
Merge branch 'main' of github.com:coder/coder into spike/1375_in_proc…
spikecurtis May 19, 2022
c5aaa82
Merge branch 'main' of github.com:coder/coder into spike/1375_in_proc…
spikecurtis May 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cli/agent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ func TestWorkspaceAgent(t *testing.T) {
instanceID := "instanceidentifier"
certificates, metadataClient := coderdtest.NewAzureInstanceIdentity(t, instanceID)
client := coderdtest.New(t, &coderdtest.Options{
AzureCertificates: certificates,
AzureCertificates: certificates,
IncludeProvisionerD: true,
})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
Provision: []*proto.Provision_Response{{
Expand Down Expand Up @@ -73,10 +73,10 @@ func TestWorkspaceAgent(t *testing.T) {
instanceID := "instanceidentifier"
certificates, metadataClient := coderdtest.NewAWSInstanceIdentity(t, instanceID)
client := coderdtest.New(t, &coderdtest.Options{
AWSCertificates: certificates,
AWSCertificates: certificates,
IncludeProvisionerD: true,
})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
Provision: []*proto.Provision_Response{{
Expand Down Expand Up @@ -128,9 +128,9 @@ func TestWorkspaceAgent(t *testing.T) {
validator, metadata := coderdtest.NewGoogleInstanceIdentity(t, instanceID, false)
client := coderdtest.New(t, &coderdtest.Options{
GoogleTokenValidator: validator,
IncludeProvisionerD: true,
})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
Provision: []*proto.Provision_Response{{
Expand Down
15 changes: 5 additions & 10 deletions cli/autostart_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,7 @@ func TestAutostart(t *testing.T) {

var (
ctx = context.Background()
client = coderdtest.New(t, nil)
_ = coderdtest.NewProvisionerDaemon(t, client)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
Expand Down Expand Up @@ -54,8 +53,7 @@ func TestAutostart(t *testing.T) {

var (
ctx = context.Background()
client = coderdtest.New(t, nil)
_ = coderdtest.NewProvisionerDaemon(t, client)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
Expand Down Expand Up @@ -99,8 +97,7 @@ func TestAutostart(t *testing.T) {
t.Parallel()

var (
client = coderdtest.New(t, nil)
_ = coderdtest.NewProvisionerDaemon(t, client)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
Expand All @@ -117,8 +114,7 @@ func TestAutostart(t *testing.T) {
t.Parallel()

var (
client = coderdtest.New(t, nil)
_ = coderdtest.NewProvisionerDaemon(t, client)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
Expand All @@ -136,8 +132,7 @@ func TestAutostart(t *testing.T) {

var (
ctx = context.Background()
client = coderdtest.New(t, nil)
_ = coderdtest.NewProvisionerDaemon(t, client)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
version = coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
Expand Down
3 changes: 1 addition & 2 deletions cli/configssh_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,8 @@ import (
func TestConfigSSH(t *testing.T) {
t.Parallel()

client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
authToken := uuid.NewString()
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
Expand Down
12 changes: 4 additions & 8 deletions cli/create_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ func TestCreate(t *testing.T) {
t.Parallel()
t.Run("Create", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
Expand Down Expand Up @@ -48,9 +47,8 @@ func TestCreate(t *testing.T) {

t.Run("CreateFromList", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
_ = coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
Expand Down Expand Up @@ -79,9 +77,8 @@ func TestCreate(t *testing.T) {

t.Run("FromNothing", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
_ = coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
Expand Down Expand Up @@ -111,9 +108,8 @@ func TestCreate(t *testing.T) {

t.Run("WithParameter", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)

defaultValue := "something"
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Expand Down
5 changes: 2 additions & 3 deletions cli/gitssh_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
func TestGitSSH(t *testing.T) {
t.Parallel()
t.Run("Dial", func(t *testing.T) {
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)

// get user public key
Expand All @@ -31,9 +31,8 @@ func TestGitSSH(t *testing.T) {
publicKey, _, _, _, err := gossh.ParseAuthorizedKey([]byte(keypair.PublicKey))
require.NoError(t, err)

// setup provisioner
// setup template
agentToken := uuid.NewString()
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionDryRun: echo.ProvisionComplete,
Expand Down
3 changes: 1 addition & 2 deletions cli/list_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,8 @@ func TestList(t *testing.T) {
t.Parallel()
t.Run("Single", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
Expand Down
11 changes: 5 additions & 6 deletions cli/portforward_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func TestPortForward(t *testing.T) {
t.Run("OnePort", func(t *testing.T) {
t.Parallel()
var (
client = coderdtest.New(t, nil)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
_, workspace = runAgent(t, client, user.UserID)
l1, p1 = setupTestListener(t, c.setupRemote(t))
Expand Down Expand Up @@ -193,7 +193,7 @@ func TestPortForward(t *testing.T) {
t.Run("TwoPorts", func(t *testing.T) {
t.Parallel()
var (
client = coderdtest.New(t, nil)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
_, workspace = runAgent(t, client, user.UserID)
l1, p1 = setupTestListener(t, c.setupRemote(t))
Expand Down Expand Up @@ -244,7 +244,7 @@ func TestPortForward(t *testing.T) {
t.Run("TCP2Unix", func(t *testing.T) {
t.Parallel()
var (
client = coderdtest.New(t, nil)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
_, workspace = runAgent(t, client, user.UserID)

Expand Down Expand Up @@ -297,7 +297,7 @@ func TestPortForward(t *testing.T) {
t.Run("All", func(t *testing.T) {
t.Parallel()
var (
client = coderdtest.New(t, nil)
client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user = coderdtest.CreateFirstUser(t, client)
_, workspace = runAgent(t, client, user.UserID)
// These aren't fixed size because we exclude Unix on Windows.
Expand Down Expand Up @@ -371,9 +371,8 @@ func runAgent(t *testing.T, client *codersdk.Client, userID uuid.UUID) ([]coders
require.Greater(t, len(user.OrganizationIDs), 0, "user has no organizations")
orgID := user.OrganizationIDs[0]

// Setup echo provisioner
// Setup template
agentToken := uuid.NewString()
coderdtest.NewProvisionerDaemon(t, client)
version := coderdtest.CreateTemplateVersion(t, client, orgID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionDryRun: echo.ProvisionComplete,
Expand Down
40 changes: 28 additions & 12 deletions cli/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
"path/filepath"
"time"

"github.com/coder/coder/provisioner/echo"

"github.com/briandowns/spinner"
"github.com/coreos/go-systemd/daemon"
"github.com/google/go-github/v43/github"
Expand Down Expand Up @@ -249,7 +251,7 @@ func server() *cobra.Command {
}
}

handler, closeCoderd := coderd.New(options)
coderDaemon := coderd.New(options)
client := codersdk.New(localURL)
if tlsEnable {
// Secure transport isn't needed for locally communicating!
Expand All @@ -275,7 +277,7 @@ func server() *cobra.Command {
errCh := make(chan error, 1)
provisionerDaemons := make([]*provisionerd.Server, 0)
for i := 0; uint8(i) < provisionerDaemonCount; i++ {
daemonClose, err := newProvisionerDaemon(cmd.Context(), client, logger, cacheDir, errCh)
daemonClose, err := newProvisionerDaemon(cmd.Context(), coderDaemon, logger, cacheDir, errCh, dev)
if err != nil {
return xerrors.Errorf("create provisioner daemon: %w", err)
}
Expand All @@ -295,7 +297,7 @@ func server() *cobra.Command {
// These errors are typically noise like "TLS: EOF". Vault does similar:
// https://github.com/hashicorp/vault/blob/e2490059d0711635e529a4efcbaa1b26998d6e1c/command/server.go#L2714
ErrorLog: log.New(io.Discard, "", 0),
Handler: handler,
Handler: coderDaemon.Handler(),
BaseContext: func(_ net.Listener) context.Context {
return shutdownConnsCtx
},
Expand Down Expand Up @@ -363,15 +365,15 @@ func server() *cobra.Command {
signal.Notify(stopChan, os.Interrupt)
select {
case <-cmd.Context().Done():
closeCoderd()
coderDaemon.CloseWait()
return cmd.Context().Err()
case err := <-tunnelErrChan:
if err != nil {
return err
}
case err := <-errCh:
shutdownConns()
closeCoderd()
coderDaemon.CloseWait()
return err
case <-stopChan:
}
Expand Down Expand Up @@ -435,7 +437,7 @@ func server() *cobra.Command {

_, _ = fmt.Fprintf(cmd.OutOrStdout(), cliui.Styles.Prompt.String()+"Waiting for WebSocket connections to close...\n")
shutdownConns()
closeCoderd()
coderDaemon.CloseWait()
return nil
},
}
Expand Down Expand Up @@ -529,7 +531,9 @@ func createFirstUser(cmd *cobra.Command, client *codersdk.Client, cfg config.Roo
return nil
}

func newProvisionerDaemon(ctx context.Context, client *codersdk.Client, logger slog.Logger, cacheDir string, errChan chan error) (*provisionerd.Server, error) {
// nolint:revive
func newProvisionerDaemon(ctx context.Context, coderDaemon coderd.CoderD,
logger slog.Logger, cacheDir string, errChan chan error, dev bool) (*provisionerd.Server, error) {
err := os.MkdirAll(cacheDir, 0700)
if err != nil {
return nil, xerrors.Errorf("mkdir %q: %w", cacheDir, err)
Expand All @@ -554,14 +558,26 @@ func newProvisionerDaemon(ctx context.Context, client *codersdk.Client, logger s
return nil, err
}

return provisionerd.New(client.ListenProvisionerDaemon, &provisionerd.Options{
provisioners := provisionerd.Provisioners{
string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(provisionersdk.Conn(terraformClient)),
}
// include echo provisioner when in dev mode
if dev {
echoClient, echoServer := provisionersdk.TransportPipe()
go func() {
err := echo.Serve(ctx, &provisionersdk.ServeOptions{Listener: echoServer})
if err != nil {
errChan <- err
}
}()
provisioners[string(database.ProvisionerTypeEcho)] = proto.NewDRPCProvisionerClient(provisionersdk.Conn(echoClient))
}
return provisionerd.New(coderDaemon.ListenProvisionerDaemon, &provisionerd.Options{
Logger: logger,
PollInterval: 500 * time.Millisecond,
UpdateInterval: 500 * time.Millisecond,
Provisioners: provisionerd.Provisioners{
string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(provisionersdk.Conn(terraformClient)),
},
WorkDirectory: tempDir,
Provisioners: provisioners,
WorkDirectory: tempDir,
}), nil
}

Expand Down
11 changes: 5 additions & 6 deletions cli/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,12 +236,11 @@ func TestServer(t *testing.T) {
}
ctx, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
root, cfg := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--provisioner-daemons", "0")
done := make(chan struct{})
root, cfg := clitest.New(t, "server", "--dev", "--tunnel=false", "--address", ":0", "--provisioner-daemons", "1")
serverErr := make(chan error)
go func() {
defer close(done)
err := root.ExecuteContext(ctx)
require.NoError(t, err)
serverErr <- err
}()
var token string
require.Eventually(t, func() bool {
Expand All @@ -258,7 +257,6 @@ func TestServer(t *testing.T) {
client.SessionToken = token
orgs, err := client.OrganizationsByUser(ctx, codersdk.Me)
require.NoError(t, err)
coderdtest.NewProvisionerDaemon(t, client)

// Create a workspace so the cleanup occurs!
version := coderdtest.CreateTemplateVersion(t, client, orgs[0].ID, nil)
Expand All @@ -278,7 +276,8 @@ func TestServer(t *testing.T) {
require.NoError(t, err)
err = currentProcess.Signal(os.Interrupt)
require.NoError(t, err)
<-done
err = <-serverErr
require.NoError(t, err)
})
t.Run("DatadogTracerNoLeak", func(t *testing.T) {
t.Parallel()
Expand Down
6 changes: 2 additions & 4 deletions cli/ssh_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,8 @@ func TestSSH(t *testing.T) {
t.Parallel()
t.Run("ImmediateExit", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
agentToken := uuid.NewString()
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
Expand Down Expand Up @@ -81,9 +80,8 @@ func TestSSH(t *testing.T) {
})
t.Run("Stdio", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
coderdtest.NewProvisionerDaemon(t, client)
agentToken := uuid.NewString()
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
Expand Down
Loading