diff --git a/agent/agentcontainers/containers.go b/agent/agentcontainers/containers.go index 4f03f35ed5710..031d3c7208424 100644 --- a/agent/agentcontainers/containers.go +++ b/agent/agentcontainers/containers.go @@ -144,6 +144,8 @@ type Lister interface { // NoopLister is a Lister interface that never returns any containers. type NoopLister struct{} +var _ Lister = NoopLister{} + func (NoopLister) List(_ context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) { return codersdk.WorkspaceAgentListContainersResponse{}, nil } diff --git a/agent/agentcontainers/containers_dockercli.go b/agent/agentcontainers/containers_dockercli.go index 3842735116329..64f264c1ba730 100644 --- a/agent/agentcontainers/containers_dockercli.go +++ b/agent/agentcontainers/containers_dockercli.go @@ -6,6 +6,9 @@ import ( "context" "encoding/json" "fmt" + "os" + "os/user" + "slices" "sort" "strconv" "strings" @@ -31,6 +34,210 @@ func NewDocker(execer agentexec.Execer) Lister { } } +// DockerEnvInfoer is an implementation of agentssh.EnvInfoer that returns +// information about a container. +type DockerEnvInfoer struct { + container string + user *user.User + userShell string + env []string +} + +// EnvInfo returns information about the environment of a container. +func EnvInfo(ctx context.Context, execer agentexec.Execer, container, containerUser string) (*DockerEnvInfoer, error) { + var dei DockerEnvInfoer + dei.container = container + + if containerUser == "" { + // Get the "default" user of the container if no user is specified. + // TODO: handle different container runtimes. + cmd, args := wrapDockerExec(container, "", "whoami") + stdout, stderr, err := run(ctx, execer, cmd, args...) + if err != nil { + return nil, xerrors.Errorf("get container user: run whoami: %w: %s", err, stderr) + } + if len(stdout) == 0 { + return nil, xerrors.Errorf("get container user: run whoami: empty output") + } + containerUser = stdout + } + // Now that we know the username, get the required info from the container. + // We can't assume the presence of `getent` so we'll just have to sniff /etc/passwd. + cmd, args := wrapDockerExec(container, containerUser, "cat", "/etc/passwd") + stdout, stderr, err := run(ctx, execer, cmd, args...) + if err != nil { + return nil, xerrors.Errorf("get container user: read /etc/passwd: %w: %q", err, stderr) + } + + scanner := bufio.NewScanner(strings.NewReader(stdout)) + var foundLine string + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, containerUser+":") { + continue + } + foundLine = line + break + } + if err := scanner.Err(); err != nil { + return nil, xerrors.Errorf("get container user: scan /etc/passwd: %w", err) + } + if foundLine == "" { + return nil, xerrors.Errorf("get container user: no matching entry for %q found in /etc/passwd", containerUser) + } + + // Parse the output of /etc/passwd. It looks like this: + // postgres:x:999:999::/var/lib/postgresql:/bin/bash + passwdFields := strings.Split(foundLine, ":") + if len(passwdFields) != 7 { + return nil, xerrors.Errorf("get container user: invalid line in /etc/passwd: %q", foundLine) + } + + // The fifth entry in /etc/passwd contains GECOS information, which is a + // comma-separated list of fields. The first field is the user's full name. + gecos := strings.Split(passwdFields[4], ",") + fullName := "" + if len(gecos) > 1 { + fullName = gecos[0] + } + + dei.user = &user.User{ + Gid: passwdFields[3], + HomeDir: passwdFields[5], + Name: fullName, + Uid: passwdFields[2], + Username: containerUser, + } + dei.userShell = passwdFields[6] + + // We need to inspect the container labels for remoteEnv and append these to + // the resulting docker exec command. + // ref: https://code.visualstudio.com/docs/devcontainers/attach-container + env, err := devcontainerEnv(ctx, execer, container) + if err != nil { // best effort. + return nil, xerrors.Errorf("read devcontainer remoteEnv: %w", err) + } + dei.env = env + + return &dei, nil +} + +func (dei *DockerEnvInfoer) CurrentUser() (*user.User, error) { + // Clone the user so that the caller can't modify it + u := *dei.user + return &u, nil +} + +func (*DockerEnvInfoer) Environ() []string { + // Return a clone of the environment so that the caller can't modify it + return os.Environ() +} + +func (*DockerEnvInfoer) UserHomeDir() (string, error) { + // We default the working directory of the command to the user's home + // directory. Since this came from inside the container, we cannot guarantee + // that this exists on the host. Return the "real" home directory of the user + // instead. + return os.UserHomeDir() +} + +func (dei *DockerEnvInfoer) UserShell(string) (string, error) { + return dei.userShell, nil +} + +func (dei *DockerEnvInfoer) ModifyCommand(cmd string, args ...string) (string, []string) { + // Wrap the command with `docker exec` and run it as the container user. + // There is some additional munging here regarding the container user and environment. + dockerArgs := []string{ + "exec", + // The assumption is that this command will be a shell command, so allocate a PTY. + "--interactive", + "--tty", + // Run the command as the user in the container. + "--user", + dei.user.Username, + // Set the working directory to the user's home directory as a sane default. + "--workdir", + dei.user.HomeDir, + } + + // Append the environment variables from the container. + for _, e := range dei.env { + dockerArgs = append(dockerArgs, "--env", e) + } + + // Append the container name and the command. + dockerArgs = append(dockerArgs, dei.container, cmd) + return "docker", append(dockerArgs, args...) +} + +// devcontainerEnv is a helper function that inspects the container labels to +// find the required environment variables for running a command in the container. +func devcontainerEnv(ctx context.Context, execer agentexec.Execer, container string) ([]string, error) { + ins, stderr, err := runDockerInspect(ctx, execer, container) + if err != nil { + return nil, xerrors.Errorf("inspect container: %w: %q", err, stderr) + } + + if len(ins) != 1 { + return nil, xerrors.Errorf("inspect container: expected 1 container, got %d", len(ins)) + } + + in := ins[0] + if in.Config.Labels == nil { + return nil, nil + } + + // We want to look for the devcontainer metadata, which is in the + // value of the label `devcontainer.metadata`. + rawMeta, ok := in.Config.Labels["devcontainer.metadata"] + if !ok { + return nil, nil + } + meta := struct { + RemoteEnv map[string]string `json:"remoteEnv"` + }{} + if err := json.Unmarshal([]byte(rawMeta), &meta); err != nil { + return nil, xerrors.Errorf("unmarshal devcontainer.metadata: %w", err) + } + + // The environment variables are stored in the `remoteEnv` key. + env := make([]string, 0, len(meta.RemoteEnv)) + for k, v := range meta.RemoteEnv { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + slices.Sort(env) + return env, nil +} + +// wrapDockerExec is a helper function that wraps the given command and arguments +// with a docker exec command that runs as the given user in the given +// container. This is used to fetch information about a container prior to +// running the actual command. +func wrapDockerExec(containerName, userName, cmd string, args ...string) (string, []string) { + dockerArgs := []string{"exec", "--interactive"} + if userName != "" { + dockerArgs = append(dockerArgs, "--user", userName) + } + dockerArgs = append(dockerArgs, containerName, cmd) + return "docker", append(dockerArgs, args...) +} + +// Helper function to run a command and return its stdout and stderr. +// We want to differentiate stdout and stderr instead of using CombinedOutput. +// We also want to differentiate between a command running successfully with +// output to stderr and a non-zero exit code. +func run(ctx context.Context, execer agentexec.Execer, cmd string, args ...string) (stdout, stderr string, err error) { + var stdoutBuf, stderrBuf strings.Builder + execCmd := execer.CommandContext(ctx, cmd, args...) + execCmd.Stdout = &stdoutBuf + execCmd.Stderr = &stderrBuf + err = execCmd.Run() + stdout = strings.TrimSpace(stdoutBuf.String()) + stderr = strings.TrimSpace(stderrBuf.String()) + return stdout, stderr, err +} + func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentListContainersResponse, error) { var stdoutBuf, stderrBuf bytes.Buffer // List all container IDs, one per line, with no truncation @@ -66,30 +273,16 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi } // now we can get the detailed information for each container - // Run `docker inspect` on each container ID - stdoutBuf.Reset() - stderrBuf.Reset() - // nolint: gosec // We are not executing user input, these IDs come from - // `docker ps`. - cmd = dcl.execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...) - cmd.Stdout = &stdoutBuf - cmd.Stderr = &stderrBuf - if err := cmd.Run(); err != nil { - return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("run docker inspect: %w: %s", err, strings.TrimSpace(stderrBuf.String())) - } - - dockerInspectStderr := strings.TrimSpace(stderrBuf.String()) - + // Run `docker inspect` on each container ID. // NOTE: There is an unavoidable potential race condition where a // container is removed between `docker ps` and `docker inspect`. // In this case, stderr will contain an error message but stdout // will still contain valid JSON. We will just end up missing // information about the removed container. We could potentially // log this error, but I'm not sure it's worth it. - ins := make([]dockerInspect, 0, len(ids)) - if err := json.NewDecoder(&stdoutBuf).Decode(&ins); err != nil { - // However, if we just get invalid JSON, we should absolutely return an error. - return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("decode docker inspect output: %w", err) + ins, dockerInspectStderr, err := runDockerInspect(ctx, dcl.execer, ids...) + if err != nil { + return codersdk.WorkspaceAgentListContainersResponse{}, xerrors.Errorf("run docker inspect: %w", err) } res := codersdk.WorkspaceAgentListContainersResponse{ @@ -111,6 +304,28 @@ func (dcl *DockerCLILister) List(ctx context.Context) (codersdk.WorkspaceAgentLi return res, nil } +// runDockerInspect is a helper function that runs `docker inspect` on the given +// container IDs and returns the parsed output. +// The stderr output is also returned for logging purposes. +func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) ([]dockerInspect, string, error) { + var stdoutBuf, stderrBuf bytes.Buffer + cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...) + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + err := cmd.Run() + stderr := strings.TrimSpace(stderrBuf.String()) + if err != nil { + return nil, stderr, err + } + + var ins []dockerInspect + if err := json.NewDecoder(&stdoutBuf).Decode(&ins); err != nil { + return nil, stderr, xerrors.Errorf("decode docker inspect output: %w", err) + } + + return ins, stderr, nil +} + // To avoid a direct dependency on the Docker API, we use the docker CLI // to fetch information about containers. type dockerInspect struct { diff --git a/agent/agentcontainers/containers_internal_test.go b/agent/agentcontainers/containers_internal_test.go index e15deae54c2bd..cdda03f9c8200 100644 --- a/agent/agentcontainers/containers_internal_test.go +++ b/agent/agentcontainers/containers_internal_test.go @@ -3,34 +3,38 @@ package agentcontainers import ( "fmt" "os" + "slices" "strconv" "strings" "testing" "time" + "go.uber.org/mock/gomock" + "github.com/google/uuid" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" "github.com/coder/coder/v2/agent/agentcontainers/acmock" "github.com/coder/coder/v2/agent/agentexec" "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/pty" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" ) -// TestDockerCLIContainerLister tests the happy path of the -// dockerCLIContainerLister.List method. It starts a container with a known +// TestIntegrationDocker tests agentcontainers functionality using a real +// Docker container. It starts a container with a known // label, lists the containers, and verifies that the expected container is -// returned. The container is deleted after the test is complete. +// returned. It also executes a sample command inside the container. +// The container is deleted after the test is complete. // As this test creates containers, it is skipped by default. // It can be run manually as follows: // // CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerCLIContainerLister -func TestDockerCLIContainerLister(t *testing.T) { +func TestIntegrationDocker(t *testing.T) { t.Parallel() if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" { t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test") @@ -44,10 +48,13 @@ func TestDockerCLIContainerLister(t *testing.T) { // Pick a random port to expose for testing port bindings. testRandPort := testutil.RandomPortNoListen(t) ct, err := pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "busybox", - Tag: "latest", - Cmd: []string{"sleep", "infnity"}, - Labels: map[string]string{"com.coder.test": testLabelValue}, + Repository: "busybox", + Tag: "latest", + Cmd: []string{"sleep", "infnity"}, + Labels: map[string]string{ + "com.coder.test": testLabelValue, + "devcontainer.metadata": `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`, + }, Mounts: []string{testTempDir + ":" + testTempDir}, ExposedPorts: []string{fmt.Sprintf("%d/tcp", testRandPort)}, PortBindings: map[docker.Port][]docker.PortBinding{ @@ -68,6 +75,11 @@ func TestDockerCLIContainerLister(t *testing.T) { assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name) t.Logf("Purged container %q", ct.Container.Name) }) + // Wait for container to start + require.Eventually(t, func() bool { + ct, ok := pool.ContainerByName(ct.Container.Name) + return ok && ct.Container.State.Running + }, testutil.WaitShort, testutil.IntervalSlow, "Container did not start in time") dcl := NewDocker(agentexec.DefaultExecer) ctx := testutil.Context(t, testutil.WaitShort) @@ -93,12 +105,93 @@ func TestDockerCLIContainerLister(t *testing.T) { if assert.Len(t, foundContainer.Volumes, 1) { assert.Equal(t, testTempDir, foundContainer.Volumes[testTempDir]) } + // Test that EnvInfo is able to correctly modify a command to be + // executed inside the container. + dei, err := EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, "") + require.NoError(t, err, "Expected no error from DockerEnvInfo()") + ptyWrappedCmd, ptyWrappedArgs := dei.ModifyCommand("/bin/sh", "--norc") + ptyCmd, ptyPs, err := pty.Start(agentexec.DefaultExecer.PTYCommandContext(ctx, ptyWrappedCmd, ptyWrappedArgs...)) + require.NoError(t, err, "failed to start pty command") + t.Cleanup(func() { + _ = ptyPs.Kill() + _ = ptyCmd.Close() + }) + tr := testutil.NewTerminalReader(t, ptyCmd.OutputReader()) + matchPrompt := func(line string) bool { + return strings.Contains(line, "#") + } + matchHostnameCmd := func(line string) bool { + return strings.Contains(strings.TrimSpace(line), "hostname") + } + matchHostnameOuput := func(line string) bool { + return strings.Contains(strings.TrimSpace(line), ct.Container.Config.Hostname) + } + matchEnvCmd := func(line string) bool { + return strings.Contains(strings.TrimSpace(line), "env") + } + matchEnvOutput := func(line string) bool { + return strings.Contains(line, "FOO=bar") || strings.Contains(line, "MULTILINE=foo") + } + require.NoError(t, tr.ReadUntil(ctx, matchPrompt), "failed to match prompt") + t.Logf("Matched prompt") + _, err = ptyCmd.InputWriter().Write([]byte("hostname\r\n")) + require.NoError(t, err, "failed to write to pty") + t.Logf("Wrote hostname command") + require.NoError(t, tr.ReadUntil(ctx, matchHostnameCmd), "failed to match hostname command") + t.Logf("Matched hostname command") + require.NoError(t, tr.ReadUntil(ctx, matchHostnameOuput), "failed to match hostname output") + t.Logf("Matched hostname output") + _, err = ptyCmd.InputWriter().Write([]byte("env\r\n")) + require.NoError(t, err, "failed to write to pty") + t.Logf("Wrote env command") + require.NoError(t, tr.ReadUntil(ctx, matchEnvCmd), "failed to match env command") + t.Logf("Matched env command") + require.NoError(t, tr.ReadUntil(ctx, matchEnvOutput), "failed to match env output") + t.Logf("Matched env output") break } } assert.True(t, found, "Expected to find container with label 'com.coder.test=%s'", testLabelValue) } +func TestWrapDockerExec(t *testing.T) { + t.Parallel() + tests := []struct { + name string + containerUser string + cmdArgs []string + wantCmd []string + }{ + { + name: "cmd with no args", + containerUser: "my-user", + cmdArgs: []string{"my-cmd"}, + wantCmd: []string{"docker", "exec", "--interactive", "--user", "my-user", "my-container", "my-cmd"}, + }, + { + name: "cmd with args", + containerUser: "my-user", + cmdArgs: []string{"my-cmd", "arg1", "--arg2", "arg3", "--arg4"}, + wantCmd: []string{"docker", "exec", "--interactive", "--user", "my-user", "my-container", "my-cmd", "arg1", "--arg2", "arg3", "--arg4"}, + }, + { + name: "no user specified", + containerUser: "", + cmdArgs: []string{"my-cmd"}, + wantCmd: []string{"docker", "exec", "--interactive", "my-container", "my-cmd"}, + }, + } + for _, tt := range tests { + tt := tt // appease the linter even though this isn't needed anymore + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + actualCmd, actualArgs := wrapDockerExec("my-container", tt.containerUser, tt.cmdArgs[0], tt.cmdArgs[1:]...) + assert.Equal(t, tt.wantCmd[0], actualCmd) + assert.Equal(t, tt.wantCmd[1:], actualArgs) + }) + } +} + // TestContainersHandler tests the containersHandler.getContainers method using // a mock implementation. It specifically tests caching behavior. func TestContainersHandler(t *testing.T) { @@ -319,6 +412,131 @@ func TestConvertDockerVolume(t *testing.T) { } } +// TestDockerEnvInfoer tests the ability of EnvInfo to extract information from +// running containers. Containers are deleted after the test is complete. +// As this test creates containers, it is skipped by default. +// It can be run manually as follows: +// +// CODER_TEST_USE_DOCKER=1 go test ./agent/agentcontainers -run TestDockerEnvInfoer +func TestDockerEnvInfoer(t *testing.T) { + t.Parallel() + if ctud, ok := os.LookupEnv("CODER_TEST_USE_DOCKER"); !ok || ctud != "1" { + t.Skip("Set CODER_TEST_USE_DOCKER=1 to run this test") + } + + pool, err := dockertest.NewPool("") + require.NoError(t, err, "Could not connect to docker") + // nolint:paralleltest // variable recapture no longer required + for idx, tt := range []struct { + image string + labels map[string]string + expectedEnv []string + containerUser string + expectedUsername string + expectedUserShell string + }{ + { + image: "busybox:latest", + labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`}, + + expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"}, + expectedUsername: "root", + expectedUserShell: "/bin/sh", + }, + { + image: "busybox:latest", + labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`}, + expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"}, + containerUser: "root", + expectedUsername: "root", + expectedUserShell: "/bin/sh", + }, + { + image: "codercom/enterprise-minimal:ubuntu", + labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`}, + expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"}, + expectedUsername: "coder", + expectedUserShell: "/bin/bash", + }, + { + image: "codercom/enterprise-minimal:ubuntu", + labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`}, + expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"}, + containerUser: "coder", + expectedUsername: "coder", + expectedUserShell: "/bin/bash", + }, + { + image: "codercom/enterprise-minimal:ubuntu", + labels: map[string]string{`devcontainer.metadata`: `{"remoteEnv": {"FOO": "bar", "MULTILINE": "foo\nbar\nbaz"}}`}, + expectedEnv: []string{"FOO=bar", "MULTILINE=foo\nbar\nbaz"}, + containerUser: "root", + expectedUsername: "root", + expectedUserShell: "/bin/bash", + }, + } { + t.Run(fmt.Sprintf("#%d", idx), func(t *testing.T) { + t.Parallel() + + // Start a container with the given image + // and environment variables + image := strings.Split(tt.image, ":")[0] + tag := strings.Split(tt.image, ":")[1] + ct, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: image, + Tag: tag, + Cmd: []string{"sleep", "infinity"}, + Labels: tt.labels, + }, func(config *docker.HostConfig) { + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + require.NoError(t, err, "Could not start test docker container") + t.Logf("Created container %q", ct.Container.Name) + t.Cleanup(func() { + assert.NoError(t, pool.Purge(ct), "Could not purge resource %q", ct.Container.Name) + t.Logf("Purged container %q", ct.Container.Name) + }) + + ctx := testutil.Context(t, testutil.WaitShort) + dei, err := EnvInfo(ctx, agentexec.DefaultExecer, ct.Container.ID, tt.containerUser) + require.NoError(t, err, "Expected no error from DockerEnvInfo()") + + u, err := dei.CurrentUser() + require.NoError(t, err, "Expected no error from CurrentUser()") + require.Equal(t, tt.expectedUsername, u.Username, "Expected username to match") + + hd, err := dei.UserHomeDir() + require.NoError(t, err, "Expected no error from UserHomeDir()") + require.NotEmpty(t, hd, "Expected user homedir to be non-empty") + + sh, err := dei.UserShell(tt.containerUser) + require.NoError(t, err, "Expected no error from UserShell()") + require.Equal(t, tt.expectedUserShell, sh, "Expected user shell to match") + + // We don't need to test the actual environment variables here. + environ := dei.Environ() + require.NotEmpty(t, environ, "Expected environ to be non-empty") + + // Test that the environment variables are present in modified command + // output. + envCmd, envArgs := dei.ModifyCommand("env") + for _, env := range tt.expectedEnv { + require.Subset(t, envArgs, []string{"--env", env}) + } + // Run the command in the container and check the output + // HACK: we remove the --tty argument because we're not running in a tty + envArgs = slices.DeleteFunc(envArgs, func(s string) bool { return s == "--tty" }) + stdout, stderr, err := run(ctx, agentexec.DefaultExecer, envCmd, envArgs...) + require.Empty(t, stderr, "Expected no stderr output") + require.NoError(t, err, "Expected no error from running command") + for _, env := range tt.expectedEnv { + require.Contains(t, stdout, env) + } + }) + } +} + func fakeContainer(t *testing.T, mut ...func(*codersdk.WorkspaceAgentDevcontainer)) codersdk.WorkspaceAgentDevcontainer { t.Helper() ct := codersdk.WorkspaceAgentDevcontainer{