feat(scaletest/terraform): add cert-manager, otel, and TLS (#9894)

johnstcn · web-flow · commit 72e8f88af380 · 2023-09-27T17:41:07.000+01:00
diff --git a/scaletest/terraform/k8s/cert-manager.tf b/scaletest/terraform/k8s/cert-manager.tf
@@ -0,0 +1,67 @@
+# Terraform configuration for cert-manaer
+
+locals {
+  cert_manager_namespace                    = "cert-manager"
+  cert_manager_helm_repo                    = "https://charts.jetstack.io"
+  cert_manager_helm_chart                   = "cert-manager"
+  cert_manager_release_name                 = "cert-manager"
+  cert_manager_chart_version                = "1.12.2"
+  cloudflare_issuer_private_key_secret_name = "cloudflare-issuer-private-key"
+}
+
+resource "kubernetes_secret" "cloudflare-api-key" {
+  metadata {
+    name      = "cloudflare-api-key-secret"
+    namespace = local.cert_manager_namespace
+  }
+  data = {
+    api-token = var.cloudflare_api_token
+  }
+}
+
+resource "kubernetes_namespace" "cert-manager-namespace" {
+  metadata {
+    name = local.cert_manager_namespace
+  }
+}
+
+resource "helm_release" "cert-manager" {
+  repository = local.cert_manager_helm_repo
+  chart      = local.cert_manager_helm_chart
+  name       = local.cert_manager_release_name
+  namespace  = kubernetes_namespace.cert-manager-namespace.metadata.0.name
+  values = [<<EOF
+installCRDs: true
+EOF
+  ]
+}
+
+resource "kubernetes_manifest" "cloudflare-cluster-issuer" {
+  manifest = {
+    apiVersion = "cert-manager.io/v1"
+    kind       = "ClusterIssuer"
+    metadata = {
+      name = "cloudflare-issuer"
+    }
+    spec = {
+      acme = {
+        email = var.cloudflare_email
+        privateKeySecretRef = {
+          name = local.cloudflare_issuer_private_key_secret_name
+        }
+        solvers = [
+          {
+            dns01 = {
+              cloudflare = {
+                apiTokenSecretRef = {
+                  name = kubernetes_secret.cloudflare-api-key.metadata.0.name
+                  key  = "api-token"
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}
diff --git a/scaletest/terraform/k8s/coder.tf b/scaletest/terraform/k8s/coder.tf
@@ -1,7 +1,7 @@
 data "google_client_config" "default" {}
 
 locals {
-  coder_url                 = var.coder_access_url == "" ? "http://${var.coder_address}" : var.coder_access_url
+  coder_url                 = var.coder_access_url
   coder_admin_email         = "admin@coder.com"
   coder_admin_user          = "coder"
   coder_helm_repo           = "https://helm.coder.com/v2"
@@ -61,20 +61,31 @@ data "kubernetes_secret" "coder_oidc" {
   }
 }
 
-# TLS needs to be provisioned manually for now.
-data "kubernetes_secret" "coder_tls" {
-  metadata {
-    namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-    name      = "${var.name}-tls"
+resource "kubernetes_manifest" "coder_certificate" {
+  manifest = {
+    apiVersion = "cert-manager.io/v1"
+    kind       = "Certificate"
+    metadata = {
+      name      = "${var.name}"
+      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+    }
+    spec = {
+      secretName = "${var.name}-tls"
+      dnsNames   = regex("https?://([^/]+)", local.coder_url)
+      issuerRef = {
+        name = kubernetes_manifest.cloudflare-cluster-issuer.manifest.metadata.name
+        kind = "ClusterIssuer"
+      }
+    }
   }
 }
 
-# Also need an OTEL collector deployed. Manual for now.
-data "kubernetes_service" "otel_collector" {
+data "kubernetes_secret" "coder_tls" {
   metadata {
     namespace = kubernetes_namespace.coder_namespace.metadata.0.name
-    name      = "otel-collector"
+    name      = "${var.name}-tls"
   }
+  depends_on = [kubernetes_manifest.coder_certificate]
 }
 
 resource "helm_release" "coder-chart" {
@@ -164,7 +175,7 @@ coder:
           name: "${data.kubernetes_secret.coder_oidc.metadata.0.name}"
     # Send OTEL traces to the cluster-local collector to sample 10%
     - name: "OTEL_EXPORTER_OTLP_ENDPOINT"
-      value: "http://${data.kubernetes_service.otel_collector.metadata.0.name}.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
+      value: "http://${kubernetes_manifest.otel-collector.manifest.metadata.name}-collector.${kubernetes_namespace.coder_namespace.metadata.0.name}.svc.cluster.local:4317"
     - name: "OTEL_TRACES_SAMPLER"
       value: parentbased_traceidratio
     - name: "OTEL_TRACES_SAMPLER_ARG"
diff --git a/scaletest/terraform/k8s/otel.tf b/scaletest/terraform/k8s/otel.tf
@@ -0,0 +1,69 @@
+# Terraform configuration for OpenTelemetry Operator
+
+locals {
+  otel_namespace              = "opentelemetry-operator-system"
+  otel_operator_helm_repo     = "https://open-telemetry.github.io/opentelemetry-helm-charts"
+  otel_operator_helm_chart    = "opentelemtry-operator"
+  otel_operator_release_name  = "opentelemetry-operator"
+  otel_operator_chart_version = "0.34.1"
+}
+
+resource "kubernetes_namespace" "otel-namespace" {
+  metadata {
+    name = local.otel_namespace
+  }
+  lifecycle {
+    ignore_changes = [timeouts, wait_for_default_service_account]
+  }
+}
+
+resource "helm_release" "otel-operator" {
+  repository = local.otel_operator_helm_repo
+  chart      = local.otel_operator_helm_chart
+  name       = local.otel_operator_release_name
+  namespace  = kubernetes_namespace.otel-namespace.metadata.0.name
+  # Default values
+  values = []
+}
+
+resource "kubernetes_manifest" "otel-collector" {
+  manifest = {
+    apiVersion = "opentelemetry.io/v1alpha1"
+    kind       = "OpenTelemetryCollector"
+    metadata = {
+      namespace = kubernetes_namespace.coder_namespace.metadata.0.name
+      name      = "otel"
+    }
+    spec = {
+      config = jsonencode({
+        receivers = {
+          otlp = {
+            protocols : {
+              grpc : {}
+              http : {}
+            }
+          }
+        }
+        exporters = {
+          googlecloud = {
+            logging = {
+              loglevel = "debug"
+            }
+          }
+        }
+        service = {
+          pipelines = {
+            traces = {
+              receivers  = ["otlp"]
+              processors = []
+              exporters  = ["logging", "googlecloud"]
+            }
+          }
+        }
+        image    = "otel/open-telemetry-collector-contrib:latest"
+        mode     = "deployment"
+        replicas = 1
+      })
+    }
+  }
+}
diff --git a/scaletest/terraform/k8s/vars.tf b/scaletest/terraform/k8s/vars.tf
@@ -207,3 +207,13 @@ variable "prometheus_remote_write_send_interval" {
   description = "Prometheus remote write interval."
   default     = "15s"
 }
+
+variable "cloudflare_api_token" {
+  description = "Cloudflare API token."
+  sensitive   = true
+}
+
+variable "cloudflare_email" {
+  description = "Cloudflare email address."
+  sensitive   = true
+}
