From d5e2fa02288783b896c16677991d93a00bda34ee Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 14 Aug 2024 09:29:24 +0100 Subject: [PATCH 1/3] chore(examples): update kubernetes devcontainer template with envbuilder provider --- .../devcontainer-kubernetes/README.md | 5 ++ .../templates/devcontainer-kubernetes/main.tf | 85 +++++++++++++++---- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/examples/templates/devcontainer-kubernetes/README.md b/examples/templates/devcontainer-kubernetes/README.md index 19f990322da51..fe73bfc993b8d 100644 --- a/examples/templates/devcontainer-kubernetes/README.md +++ b/examples/templates/devcontainer-kubernetes/README.md @@ -19,6 +19,8 @@ Provision Devcontainers as [Coder workspaces](https://coder.com/docs/workspaces) **Container Image**: This template uses the [envbuilder image](https://github.com/coder/envbuilder) to build a Devcontainer from a `devcontainer.json`. +**(Optional) Cache Registry**: Envbuilder can utilize a Docker registry as a cache to speed up workspace builds. The [envbuilder Terraform provider](https://github.com/coder/terraform-provider-envbuilder) will check the contents of the cache to determine if a prebuilt image exists. In the case of some missing layers in the registry (partial cache miss), Envbuilder can still pull previously built layers directly from the registry. + ### Authentication This template authenticates using a `~/.kube/config`, if present on the server, or via built-in authentication if the Coder provisioner is running on Kubernetes with an authorized ServiceAccount. To use another [authentication method](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#authentication), edit the template. @@ -31,6 +33,7 @@ This template provisions the following resources: - Kubernetes deployment (ephemeral) - Kubernetes persistent volume claim (persistent on `/workspaces`) +- Envbuilder cached image (optional, persistent). This template will fetch a Git repo containing a `devcontainer.json` specified by the `repo` parameter, and builds it with [`envbuilder`](https://github.com/coder/envbuilder). @@ -47,6 +50,8 @@ Edit the `devcontainer.json` instead! To speed up your builds, you can use a container registry as a cache. When creating the template, set the parameter `cache_repo`. +See the [Envbuilder Terraform Provider Examples](https://github.com/coder/terraform-provider-envbuilder/blob/main/examples/resources/envbuilder_cached_image/envbuilder_cached_image_resource.tf/) for a more complete example of how the provider works. + > [!NOTE] We recommend using a registry cache with authentication enabled. > To allow Envbuilder to authenticate with the registry cache, specify the variable `cache_repo_dockerconfig_secret` > with the name of a Kubernetes secret in the same namespace as Coder. The secret must contain the key `.dockerconfigjson`. diff --git a/examples/templates/devcontainer-kubernetes/main.tf b/examples/templates/devcontainer-kubernetes/main.tf index 68564d3fd4f63..9e1ba589cbaad 100644 --- a/examples/templates/devcontainer-kubernetes/main.tf +++ b/examples/templates/devcontainer-kubernetes/main.tf @@ -7,6 +7,9 @@ terraform { kubernetes = { source = "hashicorp/kubernetes" } + envbuilder = { + source = "coder/envbuilder" + } } } @@ -15,6 +18,7 @@ provider "kubernetes" { # Authenticate via ~/.kube/config or a Coder-specific ServiceAccount, depending on admin preferences config_path = var.use_kubeconfig == true ? "~/.kube/config" : null } +provider "envbuilder" {} data "coder_provisioner" "me" {} data "coder_workspace" "me" {} @@ -43,7 +47,6 @@ variable "namespace" { variable "cache_repo" { default = "" description = "Use a container registry as a cache to speed up builds." - sensitive = true type = string } @@ -139,20 +142,45 @@ data "kubernetes_secret" "cache_repo_dockerconfig_secret" { } locals { - deployment_name = "coder-${data.coder_workspace_owner.me.name}-${lower(data.coder_workspace.me.name)}" + deployment_name = "coder-${lower(data.coder_workspace.me.id)}" devcontainer_builder_image = data.coder_parameter.devcontainer_builder.value git_author_name = coalesce(data.coder_workspace_owner.me.full_name, data.coder_workspace_owner.me.name) git_author_email = data.coder_workspace_owner.me.email repo_url = data.coder_parameter.repo.value + # The envbuilder provider requires a key-value map of environment variables. + envbuilder_env = { + "CODER_AGENT_TOKEN" : coder_agent.main.token, + # Use the docker gateway if the access URL is 127.0.0.1 + "CODER_AGENT_URL" : replace(data.coder_workspace.me.access_url, "/localhost|127\\.0\\.0\\.1/", "host.docker.internal"), + "ENVBUILDER_GIT_URL" : local.repo_url, + # Use the docker gateway if the access URL is 127.0.0.1 + "ENVBUILDER_INIT_SCRIPT" : replace(coder_agent.main.init_script, "/localhost|127\\.0\\.0\\.1/", "host.docker.internal"), + "ENVBUILDER_FALLBACK_IMAGE" : data.coder_parameter.fallback_image.value, + "ENVBUILDER_CACHE_REPO" : var.cache_repo, + "ENVBUILDER_DOCKER_CONFIG_BASE64" : try(data.kubernetes_secret.cache_repo_dockerconfig_secret[0].data[".dockerconfigjson"], ""), + "ENVBUILDER_PUSH_IMAGE" : var.cache_repo == "" ? "" : "true", + #"ENVBUILDER_INSECURE": "true", # Uncomment if testing with an insecure registry. + } } -resource "kubernetes_persistent_volume_claim" "home" { +# Check for the presence of a prebuilt image in the cache repo +# that we can use instead. +resource "envbuilder_cached_image" "cached" { + count = var.cache_repo == "" ? 0 : data.coder_workspace.me.start_count + builder_image = local.devcontainer_builder_image + git_url = local.repo_url + cache_repo = var.cache_repo + extra_env = local.envbuilder_env + #insecure = true # Uncomment if testing with an insecure registry. +} + +resource "kubernetes_persistent_volume_claim" "workspaces" { metadata { - name = "coder-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}-home" + name = "coder-${lower(data.coder_workspace.me.id)}-workspaces" namespace = var.namespace labels = { - "app.kubernetes.io/name" = "coder-pvc" - "app.kubernetes.io/instance" = "coder-pvc-${lower(data.coder_workspace_owner.me.name)}-${lower(data.coder_workspace.me.name)}" + "app.kubernetes.io/name" = "coder-${lower(data.coder_workspace.me.id)}-workspaces" + "app.kubernetes.io/instance" = "coder-${lower(data.coder_workspace.me.id)}-workspaces" "app.kubernetes.io/part-of" = "coder" //Coder-specific labels. "com.coder.resource" = "true" @@ -173,13 +201,14 @@ resource "kubernetes_persistent_volume_claim" "home" { storage = "${data.coder_parameter.workspaces_volume_size.value}Gi" } } + # storage_class_name = "local-path" # Configure the StorageClass to use here, if required. } } resource "kubernetes_deployment" "main" { count = data.coder_workspace.me.start_count depends_on = [ - kubernetes_persistent_volume_claim.home + kubernetes_persistent_volume_claim.workspaces ] wait_for_rollout = false metadata { @@ -222,7 +251,7 @@ resource "kubernetes_deployment" "main" { container { name = "dev" - image = local.devcontainer_builder_image + image = var.cache_repo == "" ? local.devcontainer_builder_image : envbuilder_cached_image.cached.0.image image_pull_policy = "Always" security_context {} env { @@ -249,6 +278,15 @@ resource "kubernetes_deployment" "main" { name = "ENVBUILDER_CACHE_REPO" value = var.cache_repo } + env { + name = "ENVBUILDER_PUSH_IMAGE" + value = var.cache_repo == "" ? "" : "true" + } + # Uncomment the below if testing with an insecure registry. + # env { + # name = "ENVBUILDER_INSECURE" + # value = "true" + # } env { name = "ENVBUILDER_DOCKER_CONFIG_BASE64" value = try(data.kubernetes_secret.cache_repo_dockerconfig_secret[0].data[".dockerconfigjson"], "") @@ -271,16 +309,16 @@ resource "kubernetes_deployment" "main" { } } volume_mount { - mount_path = "/home/coder" - name = "home" + mount_path = "/workspaces" + name = "workspaces" read_only = false } } volume { - name = "home" + name = "workspaces" persistent_volume_claim { - claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name + claim_name = kubernetes_persistent_volume_claim.workspaces.metadata.0.name read_only = false } } @@ -357,9 +395,9 @@ resource "coder_agent" "main" { } metadata { - display_name = "Home Disk" - key = "3_home_disk" - script = "coder stat disk --path $HOME" + display_name = "Workspaces Disk" + key = "3_workspaces_disk" + script = "coder stat disk --path /workspaces" interval = 60 timeout = 1 } @@ -417,3 +455,20 @@ resource "coder_app" "code-server" { threshold = 6 } } + +resource "coder_metadata" "container_info" { + count = data.coder_workspace.me.start_count + resource_id = coder_agent.main.id + item { + key = "workspace image" + value = var.cache_repo == "" ? local.devcontainer_builder_image : envbuilder_cached_image.cached.0.image + } + item { + key = "git url" + value = local.repo_url + } + item { + key = "cache repo" + value = var.cache_repo == "" ? "not enabled" : var.cache_repo + } +} From 821ea4086eb8db487d5435d61eacada57c11131e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 14 Aug 2024 10:57:50 +0100 Subject: [PATCH 2/3] make insecure a template variable --- .../templates/devcontainer-kubernetes/main.tf | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/templates/devcontainer-kubernetes/main.tf b/examples/templates/devcontainer-kubernetes/main.tf index 9e1ba589cbaad..c45d0b435d2c9 100644 --- a/examples/templates/devcontainer-kubernetes/main.tf +++ b/examples/templates/devcontainer-kubernetes/main.tf @@ -50,6 +50,12 @@ variable "cache_repo" { type = string } +variable "insecure_cache_repo" { + default = false + description = "Enable this option if your cache registry does not serve HTTPS." + type = bool +} + data "coder_parameter" "cpu" { type = "number" name = "cpu" @@ -159,7 +165,7 @@ locals { "ENVBUILDER_CACHE_REPO" : var.cache_repo, "ENVBUILDER_DOCKER_CONFIG_BASE64" : try(data.kubernetes_secret.cache_repo_dockerconfig_secret[0].data[".dockerconfigjson"], ""), "ENVBUILDER_PUSH_IMAGE" : var.cache_repo == "" ? "" : "true", - #"ENVBUILDER_INSECURE": "true", # Uncomment if testing with an insecure registry. + "ENVBUILDER_INSECURE" : "${var.insecure_cache_repo}", } } @@ -171,7 +177,7 @@ resource "envbuilder_cached_image" "cached" { git_url = local.repo_url cache_repo = var.cache_repo extra_env = local.envbuilder_env - #insecure = true # Uncomment if testing with an insecure registry. + insecure = var.insecure_cache_repo } resource "kubernetes_persistent_volume_claim" "workspaces" { @@ -282,11 +288,10 @@ resource "kubernetes_deployment" "main" { name = "ENVBUILDER_PUSH_IMAGE" value = var.cache_repo == "" ? "" : "true" } - # Uncomment the below if testing with an insecure registry. - # env { - # name = "ENVBUILDER_INSECURE" - # value = "true" - # } + env { + name = "ENVBUILDER_INSECURE" + value = var.insecure_cache_repo + } env { name = "ENVBUILDER_DOCKER_CONFIG_BASE64" value = try(data.kubernetes_secret.cache_repo_dockerconfig_secret[0].data[".dockerconfigjson"], "") From 9d94cfc452bea79f081a1ddbbff0139086fcea2f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 15 Aug 2024 15:51:38 +0100 Subject: [PATCH 3/3] Update examples/templates/devcontainer-kubernetes/README.md Co-authored-by: Mathias Fredriksson --- examples/templates/devcontainer-kubernetes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/templates/devcontainer-kubernetes/README.md b/examples/templates/devcontainer-kubernetes/README.md index fe73bfc993b8d..b66b19c01ef84 100644 --- a/examples/templates/devcontainer-kubernetes/README.md +++ b/examples/templates/devcontainer-kubernetes/README.md @@ -19,7 +19,7 @@ Provision Devcontainers as [Coder workspaces](https://coder.com/docs/workspaces) **Container Image**: This template uses the [envbuilder image](https://github.com/coder/envbuilder) to build a Devcontainer from a `devcontainer.json`. -**(Optional) Cache Registry**: Envbuilder can utilize a Docker registry as a cache to speed up workspace builds. The [envbuilder Terraform provider](https://github.com/coder/terraform-provider-envbuilder) will check the contents of the cache to determine if a prebuilt image exists. In the case of some missing layers in the registry (partial cache miss), Envbuilder can still pull previously built layers directly from the registry. +**(Optional) Cache Registry**: Envbuilder can utilize a Docker registry as a cache to speed up workspace builds. The [envbuilder Terraform provider](https://github.com/coder/terraform-provider-envbuilder) will check the contents of the cache to determine if a prebuilt image exists. In the case of some missing layers in the registry (partial cache miss), Envbuilder can still utilize some of the build cache from the registry. ### Authentication