make k8s terraform apply somewhat cleanly

coder · johnstcn · Sep 27, 2023 · Sep 25, 2023 · Sep 25, 2023 · Sep 25, 2023
commit add8f50530b50a17a77da7da26ac9ddf2283bfbe
diff --git a/scaletest/terraform/k8s/coder.tf b/scaletest/terraform/k8s/coder.tf
@@ -1,7 +1,7 @@
 data "google_client_config" "default" {}
 
 locals {
-  coder_access_url          = "http://${var.coder_address}"
+  coder_url                 = var.coder_access_url == "" ? "http://${var.coder_address}" : var.coder_access_url
   coder_admin_email         = "admin@coder.com"
   coder_admin_user          = "coder"
   coder_helm_repo           = "https://helm.coder.com/v2"
@@ -16,6 +16,9 @@ resource "kubernetes_namespace" "coder_namespace" {
   metadata {
     name = local.coder_namespace
   }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
 }
 
 resource "random_password" "provisionerd_psk" {
@@ -31,6 +34,9 @@ resource "kubernetes_secret" "coder-db" {
   data = {
     url = var.coder_db_url
   }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
 }
 
 resource "kubernetes_secret" "provisionerd_psk" {
@@ -42,6 +48,9 @@ resource "kubernetes_secret" "provisionerd_psk" {
   data = {
     psk = random_password.provisionerd_psk.result
   }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
 }
 
 # OIDC secret needs to be manually provisioned for now.
@@ -96,10 +105,10 @@ coder:
               values:   ["${local.coder_release_name}"]
   env:
     - name: "CODER_ACCESS_URL"
-      value: "${local.coder_access_url}"
+      value: "${local.coder_url}"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
-    - name: "CODER_ENABLE_TELEMETRY"
+    - name: "CODER_TELEMETRY_ENABLE"
       value: "false"
     - name: "CODER_LOGGING_HUMAN"
       value: "/dev/null"
@@ -189,7 +198,7 @@ EOF
   ]
 }
 
-resource "helm_release" "provisionerd_chart" {
+resource "helm_release" "provisionerd-chart" {
   repository = local.coder_helm_repo
   chart      = local.provisionerd_helm_chart
   name       = local.provisionerd_release_name
@@ -217,40 +226,34 @@ coder:
               values:   ["${local.coder_release_name}"]
   env:
     - name: "CODER_URL"
-      value: "${local.coder_access_url}"
+      value: "${local.coder_url}"
+    - name: "CODER_VERBOSE"
+      value: "true"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
-    - name: "CODER_ENABLE_TELEMETRY"
+    - name: "CODER_TELEMETRY_ENABLE"
       value: "false"
     - name: "CODER_LOGGING_HUMAN"
       value: "/dev/null"
     - name: "CODER_LOGGING_STACKDRIVER"
       value: "/dev/stderr"
-    - name: "CODER_PPROF_ENABLE"
-      value: "true"
     - name: "CODER_PROMETHEUS_ENABLE"
       value: "true"
-    - name: "CODER_VERBOSE"
-      value: "true"
     - name: "CODER_PROVISIONERD_TAGS"
       value = "socpe=organization"
   image:
-    repo: ${var.coder_image_repo}
-    tag: ${var.coder_image_tag}
-  replicaCount: "${var.coder_replicas}"
+    repo: ${var.provisionerd_image_repo}
+    tag: ${var.provisionerd_image_tag}
+  replicaCount: "${var.provisionerd_replicas}"
   resources:
     requests:
-      cpu: "${var.coder_cpu_request}"
-      memory: "${var.coder_mem_request}"
+      cpu: "${var.provisionerd_cpu_request}"
+      memory: "${var.provisionerd_mem_request}"
     limits:
-      cpu: "${var.coder_cpu_limit}"
-      memory: "${var.coder_mem_limit}"
+      cpu: "${var.provisionerd_cpu_limit}"
+      memory: "${var.provisionerd_mem_limit}"
   securityContext:
     readOnlyRootFilesystem: true
-  service:
-    enable: true
-    sessionAffinity: None
-    loadBalancerIP: "${var.coder_address}"
   volumeMounts:
   - mountPath: "/tmp"
     name: cache
@@ -353,10 +356,10 @@ resource "local_file" "kubernetes_template" {
 
 resource "local_file" "output_vars" {
   filename = "${path.module}/../../.coderv2/url"
-  content  = local.coder_access_url
+  content  = local.coder_url
 }
 
 output "coder_url" {
   description = "URL of the Coder deployment"
-  value       = local.coder_access_url
+  value       = local.coder_url
 }
diff --git a/scaletest/terraform/k8s/prometheus.tf b/scaletest/terraform/k8s/prometheus.tf
@@ -14,6 +14,9 @@ resource "kubernetes_namespace" "prometheus_namespace" {
   metadata {
     name = local.prometheus_namespace
   }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
 }
 
 # Create a secret to store the remote write key
@@ -29,6 +32,9 @@ resource "kubernetes_secret" "prometheus-credentials" {
     username = var.prometheus_remote_write_user
     password = var.prometheus_remote_write_password
   }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
 }
 
 # Install Prometheus using the Bitnami Prometheus helm chart.
@@ -105,6 +111,9 @@ resource "kubernetes_secret" "prometheus-postgres-password" {
     username = var.prometheus_postgres_user
     password = var.prometheus_postgres_password
   }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_service_account_token]
+  }
 }
 
 # Install Prometheus Postgres exporter helm chart

diff --git a/scaletest/terraform/k8s/vars.tf b/scaletest/terraform/k8s/vars.tf
@@ -28,6 +28,9 @@ variable "kubernetes_nodepool_misc" {
 }
 
 // These variables control the Coder deployment.
+variable "coder_access_url" {
+  description = "Access URL for the Coder deployment."
+}
 variable "coder_replicas" {
   description = "Number of Coder replicas to provision."
   default     = 1
@@ -68,12 +71,12 @@ variable "coder_mem_limit" {
 // Allow independently scaling provisionerd resources
 variable "provisionerd_cpu_request" {
   description = "CPU request to allocate to provisionerd."
-  default     = "500m"
+  default     = "100m"
 }
 
 variable "provisionerd_mem_request" {
   description = "Memory request to allocate to provisionerd."
-  default     = "512Mi"
+  default     = "1Gi"
 }
 
 variable "provisionerd_cpu_limit" {
@@ -83,24 +86,29 @@ variable "provisionerd_cpu_limit" {
 
 variable "provisionerd_mem_limit" {
   description = "Memory limit to allocate to provisionerd."
-  default     = "1024Mi"
+  default     = "1Gi"
 }
 
 variable "provisionerd_replicas" {
   description = "Number of Provisionerd replicas."
   default     = 1
 }
 
-variable "provisionerd_concurrency" {
-  description = "Number of concurrent provisioner jobs per provisionerd instance."
-  default     = 3
-}
-
 variable "provisionerd_chart_version" {
   description = "Version of the Provisionerd Helm chart to install. Defaults to latest."
   default     = null
 }
 
+variable "provisionerd_image_repo" {
+  description = "Repository to use for Provisionerd image."
+  default     = "ghcr.io/coder/coder"
+}
+
+variable "provisionerd_image_tag" {
+  description = "Tag to use for Provisionerd image."
+  default     = "latest"
+}
+
 variable "coder_chart_version" {
   description = "Version of the Coder Helm chart to install. Defaults to latest."
   default     = null