Skip to content

Commit 353fb87

Browse files
bpmctbpmct
authored
add docs: "docker in docker" and "systemd in docker" (coder#4051)
1 parent 3e4b678 commit 353fb87

File tree

3 files changed

+301
-0
lines changed

3 files changed

+301
-0
lines changed

‎docs/images/icons/system.svg

Lines changed: 1 addition & 0 deletions
Loading

‎docs/manifest.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,12 @@
8484
"description": "Learn how to expose resource data to users",
8585
"path": "./templates/resource-metadata.md",
8686
"icon_path": "./images/icons/table-rows.svg"
87+
},
88+
{
89+
"title": "Docker in Docker",
90+
"description": "Use docker inside containerized templates",
91+
"path": "./templates/docker-in-docker.md",
92+
"icon_path": "./images/icons/docker.svg"
8793
}
8894
]
8995
},

‎docs/templates/docker-in-docker.md

Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
There are a few ways to run Docker within container-based Coder workspaces.
2+
3+
## Sysbox runtime (recommended)
4+
5+
The [Sysbox](https://github.com/nestybox/sysbox) container runtime allows unprivileged users to run system-level applications, such as Docker, securely from the workspace containers. Sysbox requires a [compatible Linux distribution](https://github.com/nestybox/sysbox/blob/master/docs/distro-compat.md) to implement these security features.
6+
7+
> Sysbox can also be used to run systemd inside Coder workspaces. See [Systemd in Docker](#systemd-in-docker).
8+
9+
### Use Sysbox in Docker-based templates:
10+
11+
After [installing Sysbox](https://github.com/nestybox/sysbox#installation) on the Coder host, modify your template to use the sysbox-runc runtime:
12+
13+
```hcl
14+
resource "docker_container" "workspace" {
15+
# ...
16+
name = "coder-${data.coder_workspace.me.owner}-${lower(data.coder_workspace.me.name)}"
17+
image = "codercom/enterprise-base:ubuntu"
18+
env = ["CODER_AGENT_TOKEN=${coder_agent.main.token}"]
19+
command = ["sh", "-c", coder_agent.main.init_script]
20+
# Use the Sysbox container runtime (required)
21+
runtime = "sysbox-runc"
22+
}
23+
24+
resource "coder_agent" "main" {
25+
arch = data.coder_provisioner.me.arch
26+
os = "linux"
27+
startup_script = <<EOF
28+
#!/bin/sh
29+
30+
# Start Docker
31+
sudo dockerd &
32+
33+
# ...
34+
EOF
35+
}
36+
```
37+
38+
### Use Sysbox in Kubernetes-based templates:
39+
40+
After [installing Sysbox on Kubernetes](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-k8s.md), modify your template to use the sysbox-runc RuntimeClass.
41+
42+
> Currently, the official [Kubernetes Terraform Provider](https://registry.terraform.io/providers/hashicorp/kubernetes/latest) does not support specifying a custom RuntimeClass. [mingfang/k8s](https://registry.terraform.io/providers/mingfang/k8s), a third-party provider, can be used instead.
43+
44+
```hcl
45+
resource "coder_agent" "main" {
46+
os = "linux"
47+
arch = "amd64"
48+
dir = "/home/coder"
49+
startup_script = <<EOF
50+
#!/bin/sh
51+
52+
# Start Docker
53+
sudo dockerd &
54+
55+
# ...
56+
EOF
57+
}
58+
59+
resource "k8s_core_v1_pod" "dev" {
60+
count = data.coder_workspace.me.start_count
61+
metadata {
62+
name = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
63+
namespace = var.workspaces_namespace
64+
annotations = {
65+
"io.kubernetes.cri-o.userns-mode" = "auto:size=65536"
66+
}
67+
}
68+
69+
# Use the Sysbox container runtime (required)
70+
runtime_class_name = "sysbox-runc
71+
72+
spec {
73+
security_context {
74+
run_asuser = 1000
75+
fsgroup = 1000
76+
}
77+
containers {
78+
name = "dev"
79+
env {
80+
name = "CODER_AGENT_TOKEN"
81+
value = coder_agent.main.token
82+
}
83+
image = "codercom/enterprise-base:ubuntu"
84+
command = ["sh", "-c", coder_agent.main.init_script]
85+
}
86+
}
87+
}
88+
```
89+
90+
> Sysbox CE (Community Edition) supports a maximum of 16 pods (workspaces) per node on Kubernetes. See the [Sysbox documentation](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-k8s.md#limitations) for more details.
91+
92+
## Privileged sidecar container
93+
94+
While less secure, you can attach a [privileged container](https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities) to your templates. This may come in handy if your nodes cannot run Sysbox.
95+
96+
### Use a privileged sidecar container in Docker-based templates:
97+
98+
```hcl
99+
resource "coder_agent" "main" {
100+
os = "linux"
101+
arch = "amd64"
102+
}
103+
104+
resource "docker_network" "private_network" {
105+
name = "network-${data.coder_workspace.me.id}"
106+
}
107+
108+
resource "docker_container" "dind" {
109+
image = "docker:dind"
110+
privileged = true
111+
name = "dind-${data.coder_workspace.me.id}"
112+
entrypoint = ["dockerd", "-H", "tcp://0.0.0.0:2375"]
113+
networks_advanced {
114+
name = docker_network.private_network.name
115+
}
116+
}
117+
118+
resource "docker_container" "workspace" {
119+
count = data.coder_workspace.me.start_count
120+
image = "codercom/enterprise-base:ubuntu"
121+
name = "dev-${data.coder_workspace.me.id}"
122+
command = ["sh", "-c", coder_agent.main.init_script]
123+
env = [
124+
"CODER_AGENT_TOKEN=${coder_agent.main.token}",
125+
"DOCKER_HOST=${docker_container.dind.name}:2375"
126+
]
127+
networks_advanced {
128+
name = docker_network.private_network.name
129+
}
130+
}
131+
```
132+
133+
### Use a privileged sidecar container in Kubernetes-based templates:
134+
135+
```hcl
136+
resource "coder_agent" "main" {
137+
os = "linux"
138+
arch = "amd64"
139+
}
140+
141+
resource "kubernetes_pod" "main" {
142+
count = data.coder_workspace.me.start_count
143+
metadata {
144+
name = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
145+
namespace = var.namespace
146+
}
147+
spec {
148+
# Run a privileged dind (Docker in Docker) container
149+
container {
150+
name = "docker-sidecar"
151+
image = "docker:dind"
152+
security_context {
153+
privileged = true
154+
}
155+
command = ["dockerd", "-H", "tcp://127.0.0.1:2375"]
156+
}
157+
container {
158+
name = "dev"
159+
image = "codercom/enterprise-base:ubuntu"
160+
command = ["sh", "-c", coder_agent.main.init_script]
161+
security_context {
162+
run_as_user = "1000"
163+
}
164+
env {
165+
name = "CODER_AGENT_TOKEN"
166+
value = coder_agent.main.token
167+
}
168+
# Use the Docker daemon in the "docker-sidecar" container
169+
env {
170+
name = "DOCKER_HOST"
171+
value = "localhost:2375"
172+
}
173+
}
174+
}
175+
}
176+
```
177+
178+
## Systemd in Docker
179+
180+
Additionally, [Sysbox](https://github.com/nestybox/sysbox) can be used to give workspaces full `systemd` capabilities.
181+
182+
### Use systemd in Docker-based templates:
183+
184+
After [installing Sysbox](https://github.com/nestybox/sysbox#installation) on the Coder host, modify your template to use the sysbox-runc runtime and start systemd:
185+
186+
```hcl
187+
resource "docker_container" "workspace" {
188+
image = "codercom/enterprise-base:ubuntu"
189+
name = "coder-${data.coder_workspace.me.owner}-${lower(data.coder_workspace.me.name)}"
190+
191+
# Use Sysbox container runtime (required)
192+
runtime = "sysbox-runc"
193+
# Run as root in order to start systemd (required)
194+
user = "0:0"
195+
196+
# Start systemd and the Coder agent
197+
command = ["sh", "-c", <<EOF
198+
# Start the Coder agent as the "coder" user
199+
# once systemd has started up
200+
sudo -u coder --preserve-env=CODER_AGENT_TOKEN /bin/bash -- <<-' EOT' &
201+
while [[ ! $(systemctl is-system-running) =~ ^(running|degraded) ]]
202+
do
203+
echo "Waiting for system to start... $(systemctl is-system-running)"
204+
sleep 2
205+
done
206+
${coder_agent.main.init_script}
207+
EOT
208+
209+
exec /sbin/init
210+
EOF
211+
,
212+
]
213+
env = ["CODER_AGENT_TOKEN=${coder_agent.main.token}"]
214+
}
215+
216+
resource "coder_agent" "main" {
217+
arch = data.coder_provisioner.me.arch
218+
os = "linux"
219+
}
220+
```
221+
222+
### Use systemd in Kubernetes-based templates:
223+
224+
After [installing Sysbox on Kubernetes](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-k8s.md), modify your template to use the sysbox-runc RuntimeClass.
225+
226+
> Currently, the official [Kubernetes Terraform Provider](https://registry.terraform.io/providers/hashicorp/kubernetes/latest) does not support specifying a custom RuntimeClass. [mingfang/k8s](https://registry.terraform.io/providers/mingfang/k8s), a third-party provider, can be used instead.
227+
228+
```hcl
229+
terraform {
230+
required_providers {
231+
coder = {
232+
source = "coder/coder"
233+
}
234+
k8s = {
235+
source = "mingfang/k8s"
236+
}
237+
}
238+
}
239+
240+
241+
resource "coder_agent" "main" {
242+
os = "linux"
243+
arch = "amd64"
244+
dir = "/home/coder"
245+
}
246+
247+
resource "k8s_core_v1_pod" "dev" {
248+
count = data.coder_workspace.me.start_count
249+
metadata {
250+
name = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
251+
namespace = var.workspaces_namespace
252+
annotations = {
253+
"io.kubernetes.cri-o.userns-mode" = "auto:size=65536"
254+
}
255+
}
256+
257+
258+
spec {
259+
260+
# Use Sysbox container runtime (required)
261+
runtime_class_name = "sysbox-runc"
262+
263+
# Run as root in order to start systemd (required)
264+
security_context {
265+
run_asuser = 0
266+
fsgroup = 0
267+
}
268+
269+
containers {
270+
name = "dev"
271+
env {
272+
name = "CODER_AGENT_TOKEN"
273+
value = coder_agent.main.token
274+
}
275+
image = "codercom/enterprise-base:ubuntu"
276+
command = ["sh", "-c", <<EOF
277+
# Start the Coder agent as the "coder" user
278+
# once systemd has started up
279+
sudo -u coder --preserve-env=CODER_AGENT_TOKEN /bin/bash -- <<-' EOT' &
280+
while [[ ! $(systemctl is-system-running) =~ ^(running|degraded) ]]
281+
do
282+
echo "Waiting for system to start... $(systemctl is-system-running)"
283+
sleep 2
284+
done
285+
${coder_agent.main.init_script}
286+
EOT
287+
288+
exec /sbin/init
289+
EOF
290+
]
291+
}
292+
}
293+
}
294+
```

0 commit comments

Comments
 (0)