diff --git a/docs/templates/docker-in-workspaces.md b/docs/templates/docker-in-workspaces.md index 42e61fa05492f..77c6ccb21595d 100644 --- a/docs/templates/docker-in-workspaces.md +++ b/docs/templates/docker-in-workspaces.md @@ -2,11 +2,12 @@ There are a few ways to run Docker within container-based Coder workspaces. -| Method | Description | Limitations | -| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Sysbox container runtime](#sysbox-container-runtime) | Install the sysbox runtime on your Kubernetes nodes for secure docker-in-docker and systemd-in-docker. Works with GKE, EKS, AKS. | Requires [compatible nodes](https://github.com/nestybox/sysbox#host-requirements). Max of 16 sysbox pods per node. [See all](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/limitations.md) | -| [Rootless Podman](#rootless-podman) | Run podman inside Coder workspaces. Does not require a custom runtime or privileged containers. Works with GKE, EKS, AKS, RKE, OpenShift | Requires smarter-device-manager for FUSE mounts. [See all](https://github.com/containers/podman/blob/main/rootless.md#shortcomings-of-rootless-podman) | -| [Privileged docker sidecar](#privileged-sidecar-container) | Run docker as a privileged sidecar container. | Requires a privileged container. Workspaces can break out to root on the host machine. | +| Method | Description | Limitations | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Sysbox container runtime](#sysbox-container-runtime) | Install the sysbox runtime on your Kubernetes nodes for secure docker-in-docker and systemd-in-docker. Works with GKE, EKS, AKS. | Requires [compatible nodes](https://github.com/nestybox/sysbox#host-requirements). Max of 16 sysbox pods per node. [See all](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/limitations.md) | +| [Envbox](#envbox) | A container image with all the packages necessary to run an inner sysbox container. Removes the need to setup sysbox-runc on your nodes. Works with GKE, EKS, AKS. | Requires running the outer container as privileged (the inner container that acts as the workspace is locked down). Requires compatible [nodes](https://github.com/nestybox/sysbox/blob/master/docs/distro-compat.md#sysbox-distro-compatibility). | +| [Rootless Podman](#rootless-podman) | Run podman inside Coder workspaces. Does not require a custom runtime or privileged containers. Works with GKE, EKS, AKS, RKE, OpenShift | Requires smarter-device-manager for FUSE mounts. [See all](https://github.com/containers/podman/blob/main/rootless.md#shortcomings-of-rootless-podman) | +| [Privileged docker sidecar](#privileged-sidecar-container) | Run docker as a privileged sidecar container. | Requires a privileged container. Workspaces can break out to root on the host machine. | ## Sysbox container runtime @@ -110,6 +111,63 @@ resource "kubernetes_pod" "dev" { > Sysbox CE (Community Edition) supports a maximum of 16 pods (workspaces) per node on Kubernetes. See the [Sysbox documentation](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-k8s.md#limitations) for more details. +## Envbox + +[Envbox](https://github.com/coder/envbox) is an image developed and maintained by Coder that bundles the sysbox runtime. It works +by starting an outer container that manages the various sysbox daemons and spawns an unprivileged +inner container that acts as the user's workspace. The inner container is able to run system-level +software similar to a regular virtual machine (e.g. `systemd`, `dockerd`, etc). Envbox offers the +following benefits over running sysbox directly on the nodes: + +- No custom runtime installation or management on your Kubernetes nodes. +- No limit to the number of pods that run envbox. + +Some drawbacks include: + +- The outer container must be run as privileged + - Note: the inner container is _not_ privileged. For more information on the security of sysbox + containers see sysbox's [official documentation](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/security.md). +- Initial workspace startup is slower than running `sysbox-runc` directly on the nodes. This is due + to `envbox` having to pull the image to its own Docker cache on its initial startup. Once the image + is cached in `envbox`, startup performance is similar. + +Envbox requires the same kernel requirements as running sysbox directly on the nodes. Refer +to sysbox's [compatibility matrix](https://github.com/nestybox/sysbox/blob/master/docs/distro-compat.md#sysbox-distro-compatibility) to ensure your nodes are compliant. + +To get started with `envbox` check out the [starter template](../../examples/templates/envbox) or visit the [repo](https://github.com/coder/envbox). + +### Authenticating with a Private Registry + +Authenticating with a private container registry can be done by referencing the credentials +via the `CODER_IMAGE_PULL_SECRET` environment variable. It is encouraged to populate this +[environment variable](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data) by using a Kubernetes [secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials). + +Refer to your container registry documentation to understand how to best create this secret. + +The following shows a minimal example using a the JSON API key from a GCP service account to pull +a private image: + +```bash +# Create the secret +$ kubectl create secret docker-registry \ + --docker-server=us.gcr.io \ + --docker-username=_json_key \ + --docker-password="$(cat ./json-key-file.yaml)" \ + --docker-email= +``` + +```hcl +env { + name = "CODER_IMAGE_PULL_SECRET" + value_from { + secret_key_ref { + name = "" + key = ".dockerconfigjson" + } + } +} +``` + ## Rootless podman [Podman](https://docs.podman.io/en/latest/) is Docker alternative that is compatible with OCI containers specification. which can run rootless inside Kubernetes pods. No custom RuntimeClass is required. diff --git a/examples/templates/envbox/README.md b/examples/templates/envbox/README.md new file mode 100644 index 0000000000000..bea44c48bc6b0 --- /dev/null +++ b/examples/templates/envbox/README.md @@ -0,0 +1,32 @@ +# envbox + +## Introduction + +`envbox` is an image that enables creating non-privileged containers capable of running system-level software (e.g. `dockerd`, `systemd`, etc) in Kubernetes. + +It mainly acts as a wrapper for the excellent [sysbox runtime](https://github.com/nestybox/sysbox/) developed by [Nestybox](https://www.nestybox.com/). For more details on the security of `sysbox` containers see sysbox's [official documentation](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/security.md). + +## Envbox Configuration + +The following environment variables can be used to configure various aspects of the inner and outer container. + +| env | usage | required | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `CODER_INNER_IMAGE` | The image to use for the inner container. | True | +| `CODER_INNER_USERNAME` | The username to use for the inner container. | True | +| `CODER_AGENT_TOKEN` | The [Coder Agent](https://coder.com/docs/v2/latest/about/architecture#agents) token to pass to the inner container. | True | +| `CODER_INNER_ENVS` | The environment variables to pass to the inner container. A wildcard can be used to match a prefix. Ex: `CODER_INNER_ENVS=KUBERNETES_*,MY_ENV,MY_OTHER_ENV` | false | +| `CODER_INNER_HOSTNAME` | The hostname to use for the inner container. | false | +| `CODER_IMAGE_PULL_SECRET` | The docker credentials to use when pulling the inner container. The recommended way to do this is to create an [Image Pull Secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials) and then reference the secret using an [environment variable](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data). | false | +| `CODER_DOCKER_BRIDGE_CIDR` | The bridge CIDR to start the Docker daemon with. | false | +| `CODER_MOUNTS` | A list of mounts to mount into the inner container. Mounts default to `rw`. Ex: `CODER_MOUNTS=/home/coder:/home/coder,/var/run/mysecret:/var/run/mysecret:ro` | false | +| `CODER_USR_LIB_DIR` | The mountpoint of the host `/usr/lib` directory. Only required when using GPUs. | false | +| `CODER_ADD_TUN` | If `CODER_ADD_TUN=true` add a TUN device to the inner container. | false | +| `CODER_ADD_FUSE` | If `CODER_ADD_FUSE=true` add a FUSE device to the inner container. | false | +| `CODER_ADD_GPU` | If `CODER_ADD_GPU=true` add detected GPUs and related files to the inner container. Requires setting `CODER_USR_LIB_DIR` and mounting in the hosts `/usr/lib/` directory. | false | +| `CODER_CPUS` | Dictates the number of CPUs to allocate the inner container. It is recommended to set this using the Kubernetes [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables). | false | +| `CODER_MEMORY` | Dictates the max memory (in bytes) to allocate the inner container. It is recommended to set this using the Kubernetes [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables). | false | + +## Contributions + +Contributions are welcome and can be made against the [envbox repo](https://github.com/coder/envbox). diff --git a/examples/templates/envbox/main.tf b/examples/templates/envbox/main.tf new file mode 100644 index 0000000000000..472a8f6682304 --- /dev/null +++ b/examples/templates/envbox/main.tf @@ -0,0 +1,302 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + version = "0.6.12" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.12.1" + } + } +} + +data "coder_parameter" "home_disk" { + name = "Disk Size" + description = "How large should the disk storing the home directory be?" + icon = "https://cdn-icons-png.flaticon.com/512/2344/2344147.png" + type = "number" + default = 10 + mutable = true + validation { + min = 10 + max = 100 + } +} + +variable "use_kubeconfig" { + type = bool + sensitive = true + description = <<-EOF + Use host kubeconfig? (true/false) + Set this to false if the Coder host is itself running as a Pod on the same + Kubernetes cluster as you are deploying workspaces to. + Set this to true if the Coder host is running outside the Kubernetes cluster + for workspaces. A valid "~/.kube/config" must be present on the Coder host. + EOF +} + +variable "namespace" { + type = string + sensitive = true + description = "The namespace to create workspaces in (must exist prior to creating workspaces)" +} + +variable "create_tun" { + type = bool + sensitive = true + description = "Add a TUN device to the workspace." +} + +variable "create_fuse" { + type = bool + description = "Add a FUSE device to the workspace." + sensitive = true +} + +variable "max_cpus" { + type = string + sensitive = true + description = "Max number of CPUs the workspace may use (e.g. 2)." +} + +variable "min_cpus" { + type = string + sensitive = true + description = "Minimum number of CPUs the workspace may use (e.g. .1)." +} + +variable "max_memory" { + type = string + description = "Maximum amount of memory to allocate the workspace (in GB)." + sensitive = true +} + +variable "min_memory" { + type = string + description = "Minimum amount of memory to allocate the workspace (in GB)." + sensitive = true +} + +provider "kubernetes" { + # Authenticate via ~/.kube/config or a Coder-specific ServiceAccount, depending on admin preferences + config_path = var.use_kubeconfig == true ? "~/.kube/config" : null +} + +data "coder_workspace" "me" {} + +resource "coder_agent" "main" { + os = "linux" + arch = "amd64" + startup_script = <