Skip to content

coder_metadata breaks Kubernetes-based templates #3721

Closed
coder/terraform-provider-coder
#49
@bpmct

Description

@bpmct

@sharkymark discovered that coder_metadata does not work with the kubernetes_pod resource (and perhaps others that follow a similar pattern).

Despite a successful apply, the workspace state is build appears as "failed" and the agent fails to connect

2022-08-28 13:18:25.287 [INFO]  <./agent/agent.go:132>  (*agent).run    connecting
2022-08-28 13:18:25.316 [WARN]  <./agent/agent.go:141>  (*agent).run    failed to dial{"error": "GET https://coder.bpmct.net/api/v2/workspaceagents/me/listen: unexpected status code 401: Agent token is invalid.: Try logging in using 'coder login \u003curl\u003e'."}
Steps to reproduce
  1. Create a template with kubernetes_pod and coder_metadata resouce. Here's the one that failed:
  terraform {
    required_providers {
      coder = {
        source  = "coder/coder"
        version = "~> 0.4.9"
      }
      kubernetes = {
        source  = "hashicorp/kubernetes"
        version = "~> 2.12.1"
      }
    }
  }
  variable "use_kubeconfig" {
    type        = bool
    sensitive   = true
    description = <<-EOF
    Use host kubeconfig? (true/false)
    Set this to false if the Coder host is itself running as a Pod on the same
    Kubernetes cluster as you are deploying workspaces to.
    Set this to true if the Coder host is running outside the Kubernetes cluster
    for workspaces.  A valid "~/.kube/config" must be present on the Coder host.
    EOF
  }
  variable "dotfiles_uri" {
    description = <<-EOF
    Dotfiles repo URI (optional)
    see https://dotfiles.github.io
    EOF
    default     = ""
  }
  variable "image" {
    description = <<-EOF
    Container image with Jupyter Lab
    EOF
    default     = "codercom/enterprise-jupyter:ubuntu"
    validation {
      condition = contains([
        "marktmilligan/jupyterlab:latest",
        "codercom/enterprise-jupyter:ubuntu"
      ], var.image)
      error_message = "Invalid image!"
    }
  }
  variable "repo" {
    description = <<-EOF
    Code repository to clone
    EOF
    default     = "sharkymark/pandas_automl.git"
    validation {
      condition = contains([
        "sharkymark/pandas_automl.git"
      ], var.repo)
      error_message = "Invalid repo!"
    }
  }
  variable "extension" {
    description = "VS Code extension"
    default     = "ms-toolsai.jupyter"
    validation {
      condition = contains([
        "ms-python.python",
        "ms-toolsai.jupyter"
      ], var.extension)
      error_message = "Invalid VS Code extension!"
    }
  }
  locals {
    jupyter-type-arg = var.jupyter == "notebook" ? "Notebook" : "Server"
  }
  variable "jupyter" {
    description = "Jupyter IDE type"
    default     = "lab"
    validation {
      condition = contains([
        "notebook",
        "lab",
      ], var.jupyter)
      error_message = "Invalid Jupyter!"
    }
  }
  variable "cpu" {
    description = "CPU (__ cores)"
    default     = 1
    validation {
      condition = contains([
        "1",
        "2",
        "4",
        "6"
      ], var.cpu)
      error_message = "Invalid cpu!"
    }
  }
  variable "memory" {
    description = "Memory (__ GB)"
    default     = 2
    validation {
      condition = contains([
        "1",
        "2",
        "4",
        "8"
      ], var.memory)
      error_message = "Invalid memory!"
    }
  }
  variable "disk_size" {
    description = "Disk size (__ GB)"
    default     = 10
  }
  variable "workspaces_namespace" {
    type        = string
    sensitive   = true
    description = "The namespace to create workspaces in (must exist prior to creating workspaces)"
    default     = "coder-workspaces"
  }
  provider "kubernetes" {
    # Authenticate via ~/.kube/config or a Coder-specific ServiceAccount, depending on admin preferences
    config_path = var.use_kubeconfig == true ? "~/.kube/config" : null
  }
  data "coder_workspace" "me" {}
  resource "coder_agent" "coder" {
    os             = "linux"
    arch           = "amd64"
    dir            = "/home/coder"
    startup_script = <<EOT
  #!/bin/bash
  # start jupyter 
  jupyter ${var.jupyter} --no-browser --${local.jupyter-type-arg}App.token='' --ip='*' --${local.jupyter-type-arg}App.base_url=/@${data.coder_workspace.me.owner}/${lower(data.coder_workspace.me.name)}/apps/jupyter-${var.jupyter}/ &
  # install code-server
  curl -fsSL https://code-server.dev/install.sh | sh
  code-server --auth none --port 13337 &
  # add some Python libraries
  pip3 install --user pandas
  # use coder CLI to clone and install dotfiles
  coder dotfiles -y ${var.dotfiles_uri}
  # clone repo
  mkdir -p ~/.ssh
  ssh-keyscan -t ed25519 github.com >> ~/.ssh/known_hosts
  git clone --progress git@github.com:${var.repo} 
  # install VS Code extension into code-server
  SERVICE_URL=https://open-vsx.org/vscode/gallery ITEM_URL=https://open-vsx.org/vscode/item code-server --install-extension ${var.extension}
  EOT
  }
  # code-server
  resource "coder_app" "code-server" {
    agent_id      = coder_agent.coder.id
    name          = "code-server"
    icon          = "/icon/code.svg"
    url           = "http://localhost:13337?folder=/home/coder"
    relative_path = true
  }
  resource "coder_app" "jupyter" {
    agent_id      = coder_agent.coder.id
    name          = "jupyter-${var.jupyter}"
    icon          = "/icon/jupyter.svg"
    url           = "http://localhost:8888/@${data.coder_workspace.me.owner}/${lower(data.coder_workspace.me.name)}/apps/jupyter-${var.jupyter}/"
    relative_path = true
  }
  resource "kubernetes_pod" "main" {
    count = data.coder_workspace.me.start_count
    metadata {
      name      = "coder-${data.coder_workspace.me.owner}-${lower(data.coder_workspace.me.name)}"
      namespace = var.workspaces_namespace
    }
    spec {
      security_context {
        run_as_user = "1000"
        fs_group    = "1000"
      }
      container {
        name              = "jupyterlab"
        image             = "docker.io/${var.image}"
        command           = ["sh", "-c", coder_agent.coder.init_script]
        image_pull_policy = "Always"
        security_context {
          run_as_user = "1000"
        }
        env {
          name  = "CODER_AGENT_TOKEN"
          value = coder_agent.coder.token
        }
        resources {
          requests = {
            cpu    = "250m"
            memory = "1000Mi"
          }
          limits = {
            cpu    = "${var.cpu}"
            memory = "${var.memory}G"
          }
        }
        volume_mount {
          mount_path = "/home/coder"
          name       = "home-directory"
        }
      }
      volume {
        name = "home-directory"
        persistent_volume_claim {
          claim_name = kubernetes_persistent_volume_claim.home-directory.metadata.0.name
        }
      }
    }
  }
  resource "kubernetes_persistent_volume_claim" "home-directory" {
    metadata {
      name      = "home-coder-${data.coder_workspace.me.owner}-${lower(data.coder_workspace.me.name)}"
      namespace = var.workspaces_namespace
    }
    spec {
      access_modes = ["ReadWriteOnce"]
      resources {
        requests = {
          storage = "${var.disk_size}Gi"
        }
      }
    }
  }
  resource "coder_metadata" "workspace_info" {
    count       = data.coder_workspace.me.start_count
    resource_id = kubernetes_pod.main[0].id
    item {
      key   = "CPU cores"
      value = "${var.cpu}G"
    }
    item {
      key   = "memory"
      value = "${var.memory}G"
    }
    item {
      key   = "Kubernetes namespace"
      value = var.workspaces_namespace
    }
    item {
      key   = "Container image"
      value = var.image
    }
    item {
      key   = "Source Code repo"
      value = var.repo
    }
    item {
      key   = "VS Code extension"
      value = var.extension
    }
    item {
      key   = "Source Code repo"
      value = var.repo
    }
  }
  1. Create a workspace with that template. Observe the build fails and the pod logs indicate the agent has an invalid token
  2. Remove the coder_metadata block and update with coder templates push
  3. Observe workspaces succeed to build.

Metadata

Metadata

Assignees

Labels

No labels
No labels

Type

No type

Projects

No projects

Milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions