add k8s

coder · bpmct · Sep 19, 2022 · Sep 13, 2022 · Sep 14, 2022 · Sep 14, 2022
commit c523571ca9f01f6575ebd0ab28c6bdeab8e3ff11
diff --git a/docs/templates/docker-in-docker.md b/docs/templates/docker-in-docker.md
@@ -1,8 +1,10 @@
-I am not really sure what to put here yet.
+There are a few ways to run Docker within container-based Coder workspaces.
 
 ## Sysbox runtime (recommended)
 
-The Sysbox container runtime allows unprivileged users to run system-level applications, such as Docker, securely from the workspace containers. Sysbox requires a [compatible Linux distribution](https://github.com/nestybox/sysbox/blob/master/docs/distro-compat.md) to implement these security features.
+The [Sysbox](https://github.com/nestybox/sysbox) container runtime allows unprivileged users to run system-level applications, such as Docker, securely from the workspace containers. Sysbox requires a [compatible Linux distribution](https://github.com/nestybox/sysbox/blob/master/docs/distro-compat.md) to implement these security features.
+
+> Sysbox can also be used to run systemd inside Coder workspaces. See [Systemd in Docker](./systemd-in-docker.md).
 
 ### Use Sysbox in Docker-based templates:
 
@@ -13,9 +15,10 @@ resource "docker_container" "workspace" {
   # ...
   name    = "coder-${data.coder_workspace.me.owner}-${lower(data.coder_workspace.me.name)}"
   image   = "codercom/enterprise-base:ubuntu"
-  runtime = "sysbox-runc"
   env     = ["CODER_AGENT_TOKEN=${coder_agent.main.token}"]
   command = ["sh", "-c", coder_agent.main.init_script]
+  # Use the Sysbox container runtime (required)
+  runtime = "sysbox-runc"
 }
 
 resource "coder_agent" "main" {
@@ -24,22 +27,70 @@ resource "coder_agent" "main" {
   startup_script = <<EOF
     #!/bin/sh
 
-    # Start docker (you can also use `dockerd&`)
-    sudo service docker start
+    # Start Docker
+    sudo dockerd &
 
     # ...
     EOF
 }
 ```
 
-> Sysbox can also be used to run systemd inside Coder workspaces. See [Systemd in Docker](./systemd-in-docker.md)
-
 ### Use Sysbox in Kubernetes-based templates:
 
+After [installing Sysbox on Kubernetes](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-k8s.md), modify your template to use the sysbox-runc RuntimeClass.
+
+> Currently, the official [Kubernetes Terraform Provider](https://registry.terraform.io/providers/hashicorp/kubernetes/latest) does not support specifying a custom RuntimeClass. [mingfang/k8s](https://registry.terraform.io/providers/mingfang/k8s), a third-party provider, can be used instead.
+
 ```hcl
+resource "coder_agent" "main" {
+  os   = "linux"
+  arch = "amd64"
+  dir  = "/home/coder"
+  startup_script = <<EOF
+    #!/bin/sh
 
+    # Start Docker
+    sudo dockerd &
+
+    # ...
+  EOF
+}
+
+resource "k8s_core_v1_pod" "dev" {
+  count = data.coder_workspace.me.start_count
+  metadata {
+    name      = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
+    namespace = var.workspaces_namespace
+    annotations = {
+      "io.kubernetes.cri-o.userns-mode" = "auto:size=65536"
+    }
+  }
+
+  # Use the Sysbox container runtime (required)
+  runtime_class_name = "sysbox-runc
+
+  spec {
+    security_context {
+      run_asuser = 1000
+      fsgroup    = 1000
+    }
+    containers {
+      name = "dev"
+      env {
+        name  = "CODER_AGENT_TOKEN"
+        value = coder_agent.main.token
+      }
+      image = "codercom/enterprise-base:ubuntu"
+      command = ["sh", "-c", coder_agent.main.init_script]
+    }
+  }
+}
 ```
 
-## Shared Docker socket
+## Privileged sidecar container (Docker and Kubernetes)
+
+TODO
+
+## Shared Docker socket (Docker only)
 
-## Privileged sidecar container
+TODO
diff --git a/docs/templates/systemd-in-docker.md b/docs/templates/systemd-in-docker.md
@@ -1,24 +1,28 @@
-I am not really sure what to put here yet.
+The [Sysbox](https://github.com/nestybox/sysbox) container runtime allows unprivileged users to run system-level applications, such as Systemd, securely from the workspace containers. Sysbox requires a [compatible Linux distribution](https://github.com/nestybox/sysbox/blob/master/docs/distro-compat.md) to implement these security features.
+
+## Docker
+
+After [installing Sysbox](https://github.com/nestybox/sysbox#installation) on the Coder host, modify your template to use the sysbox-runc runtime and start systemd:
 
 ```hcl
 resource "docker_container" "workspace" {
-  count = data.coder_workspace.me.start_count
-  image = var.docker_image
-  # Uses lower() to avoid Docker restriction on container names.
+  image = "codercom/enterprise-base:ubuntu"
   name = "coder-${data.coder_workspace.me.owner}-${lower(data.coder_workspace.me.name)}"
+
+  # Use Sysbox container runtime (required)
   runtime = "sysbox-runc"
-  env     = ["CODER_AGENT_TOKEN=${coder_agent.main.token}"]
-  # Run as root in order to start systemd
+  # Run as root in order to start systemd (required)
   user    = "0:0"
-  command = ["sh", "-c", <<EOF
 
+  # Start systemd and the Coder agent
+  command = ["sh", "-c", <<EOF
     # Start the Coder agent as the "coder" user
     # once systemd has started up
     sudo -u coder --preserve-env=CODER_AGENT_TOKEN /bin/bash -- <<-'    EOT' &
-    while [ $(systemctl is-system-running) != running ] && [ $(systemctl is-system-running) != degraded ]
+    while [[ ! $(systemctl is-system-running) =~ ^(running|degraded) ]]
     do
       echo "Waiting for system to start... $(systemctl is-system-running)"
-      sleep 1
+      sleep 2
     done
     ${coder_agent.main.init_script}
     EOT
@@ -27,10 +31,85 @@ resource "docker_container" "workspace" {
     EOF
     ,
   ]
+  env     = ["CODER_AGENT_TOKEN=${coder_agent.main.token}"]
 }
 
 resource "coder_agent" "main" {
   arch = data.coder_provisioner.me.arch
   os   = "linux"
 }
 ```
+
+## Kubernetes
+
+After [installing Sysbox on Kubernetes](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-k8s.md), modify your template to use the sysbox-runc RuntimeClass.
+
+> Currently, the official [Kubernetes Terraform Provider](https://registry.terraform.io/providers/hashicorp/kubernetes/latest) does not support specifying a custom RuntimeClass. [mingfang/k8s](https://registry.terraform.io/providers/mingfang/k8s), a third-party provider, can be used instead.
+
+```hcl
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+    }
+    k8s = {
+      source = "mingfang/k8s"
+    }
+  }
+}
+
+
+resource "coder_agent" "main" {
+  os   = "linux"
+  arch = "amd64"
+  dir  = "/home/coder"
+}
+
+resource "k8s_core_v1_pod" "dev" {
+  count = data.coder_workspace.me.start_count
+  metadata {
+    name      = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
+    namespace = var.workspaces_namespace
+    annotations = {
+      "io.kubernetes.cri-o.userns-mode" = "auto:size=65536"
+    }
+  }
+
+
+  spec {
+
+    # Use Sysbox container runtime (required)
+    runtime_class_name = "sysbox-runc"
+
+    # Run as root in order to start systemd (required)
+    security_context {
+      run_asuser = 0
+      fsgroup    = 0
+    }
+
+    containers {
+      name = "dev"
+      env {
+        name  = "CODER_AGENT_TOKEN"
+        value = coder_agent.main.token
+      }
+      image = "codercom/enterprise-base:ubuntu"
+      command = ["sh", "-c", <<EOF
+    # Start the Coder agent as the "coder" user
+    # once systemd has started up
+    sudo -u coder --preserve-env=CODER_AGENT_TOKEN /bin/bash -- <<-'    EOT' &
+    while [[ ! $(systemctl is-system-running) =~ ^(running|degraded) ]]
+    do
+      echo "Waiting for system to start... $(systemctl is-system-running)"
+      sleep 2
+    done
+    ${coder_agent.main.init_script}
+    EOT
+
+    exec /sbin/init
+    EOF
+      ]
+    }
+  }
+}
+```