Skip to content

Commit 08f0eaa

Browse files
f0sself0ssel
authored
feat: add new scaletest infrastructure (coder#15573)
Closes coder/internal#148 This has been validated to have working proxies with the `small` scenario. - [x] multi-region gcp infrastructure - [x] use cloudflare provider to automate dns entries - [x] automate proxy registration - [x] multi-regional proxies - [x] move scenarios into locals This excludes the infrastructure for `cert-manager` and `otel-collector` and those will be included in followup PRs.
1 parent d31c2f1 commit 08f0eaa

14 files changed

+1327
-0
lines changed

‎scaletest/terraform/action/cf_dns.tf

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
resource "cloudflare_record" "coder" {
2+
for_each = local.deployments
3+
zone_id = var.cloudflare_zone_id
4+
name = each.value.subdomain
5+
content = google_compute_address.coder[each.key].address
6+
type = "A"
7+
ttl = 3600
8+
}

‎scaletest/terraform/action/coder_helm_values.tftpl

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
coder:
2+
workspaceProxy: ${workspace_proxy}
3+
affinity:
4+
nodeAffinity:
5+
requiredDuringSchedulingIgnoredDuringExecution:
6+
nodeSelectorTerms:
7+
- matchExpressions:
8+
- key: "cloud.google.com/gke-nodepool"
9+
operator: "In"
10+
values: ["${node_pool}"]
11+
podAntiAffinity:
12+
preferredDuringSchedulingIgnoredDuringExecution:
13+
- weight: 1
14+
podAffinityTerm:
15+
topologyKey: "kubernetes.io/hostname"
16+
labelSelector:
17+
matchExpressions:
18+
- key: "app.kubernetes.io/instance"
19+
operator: "In"
20+
values: ["${release_name}"]
21+
env:
22+
%{~ if workspace_proxy ~}
23+
- name: "CODER_ACCESS_URL"
24+
value: "${access_url}"
25+
- name: CODER_PRIMARY_ACCESS_URL
26+
value: "${primary_url}"
27+
- name: CODER_PROXY_SESSION_TOKEN
28+
valueFrom:
29+
secretKeyRef:
30+
key: token
31+
name: "${proxy_token}"
32+
%{~ endif ~}
33+
%{~ if provisionerd ~}
34+
- name: "CODER_URL"
35+
value: "${access_url}"
36+
- name: "CODER_PROVISIONERD_TAGS"
37+
value: "scope=organization"
38+
- name: "CODER_CONFIG_DIR"
39+
value: "/tmp/config"
40+
%{~ endif ~}
41+
%{~ if !workspace_proxy && !provisionerd ~}
42+
- name: "CODER_ACCESS_URL"
43+
value: "${access_url}"
44+
- name: "CODER_PG_CONNECTION_URL"
45+
valueFrom:
46+
secretKeyRef:
47+
name: "${db_secret}"
48+
key: url
49+
- name: "CODER_PROVISIONER_DAEMONS"
50+
value: "0"
51+
- name: CODER_PROVISIONER_DAEMON_PSK
52+
valueFrom:
53+
secretKeyRef:
54+
key: psk
55+
name: "${provisionerd_psk}"
56+
- name: "CODER_PROMETHEUS_COLLECT_AGENT_STATS"
57+
value: "true"
58+
- name: "CODER_PROMETHEUS_COLLECT_DB_METRICS"
59+
value: "true"
60+
- name: "CODER_PPROF_ENABLE"
61+
value: "true"
62+
%{~ endif ~}
63+
- name: "CODER_CACHE_DIRECTORY"
64+
value: "/tmp/coder"
65+
- name: "CODER_TELEMETRY_ENABLE"
66+
value: "false"
67+
- name: "CODER_LOGGING_HUMAN"
68+
value: "/dev/null"
69+
- name: "CODER_LOGGING_STACKDRIVER"
70+
value: "/dev/stderr"
71+
- name: "CODER_PROMETHEUS_ENABLE"
72+
value: "true"
73+
- name: "CODER_VERBOSE"
74+
value: "true"
75+
- name: "CODER_EXPERIMENTS"
76+
value: "${experiments}"
77+
- name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
78+
value: "true"
79+
image:
80+
repo: ${image_repo}
81+
tag: ${image_tag}
82+
replicaCount: "${replicas}"
83+
resources:
84+
requests:
85+
cpu: "${cpu_request}"
86+
memory: "${mem_request}"
87+
limits:
88+
cpu: "${cpu_limit}"
89+
memory: "${mem_limit}"
90+
securityContext:
91+
readOnlyRootFilesystem: true
92+
%{~ if !provisionerd ~}
93+
service:
94+
enable: true
95+
sessionAffinity: None
96+
loadBalancerIP: "${ip_address}"
97+
%{~ endif ~}
98+
volumeMounts:
99+
- mountPath: "/tmp"
100+
name: cache
101+
readOnly: false
102+
volumes:
103+
- emptyDir:
104+
sizeLimit: 1024Mi
105+
name: cache

‎scaletest/terraform/action/coder_proxies.tf

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
data "http" "coder_healthy" {
2+
url = local.deployments.primary.url
3+
// Wait up to 5 minutes for DNS to propagate
4+
retry {
5+
attempts = 30
6+
min_delay_ms = 10000
7+
}
8+
9+
lifecycle {
10+
postcondition {
11+
condition = self.status_code == 200
12+
error_message = "${self.url} returned an unhealthy status code"
13+
}
14+
}
15+
16+
depends_on = [helm_release.coder_primary, cloudflare_record.coder["primary"]]
17+
}
18+
19+
resource "null_resource" "api_key" {
20+
provisioner "local-exec" {
21+
interpreter = ["/bin/bash", "-c"]
22+
command = <<EOF
23+
set -e
24+
25+
curl '${local.deployments.primary.url}/api/v2/users/first' \
26+
--data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}","username":"${local.coder_admin_user}","name":"${local.coder_admin_full_name}","trial":false}' \
27+
--insecure --silent --output /dev/null
28+
29+
session_token=$(curl '${local.deployments.primary.url}/api/v2/users/login' \
30+
--data-raw $'{"email":"${local.coder_admin_email}","password":"${local.coder_admin_password}"}' \
31+
--insecure --silent | jq -r .session_token)
32+
33+
echo -n $${session_token} > ${path.module}/.coderv2/session_token
34+
35+
api_key=$(curl '${local.deployments.primary.url}/api/v2/users/me/keys/tokens' \
36+
-H "Coder-Session-Token: $${session_token}" \
37+
--data-raw '{"token_name":"terraform","scope":"all"}' \
38+
--insecure --silent | jq -r .key)
39+
40+
echo -n $${api_key} > ${path.module}/.coderv2/api_key
41+
EOF
42+
}
43+
44+
depends_on = [data.http.coder_healthy]
45+
}
46+
47+
data "local_file" "api_key" {
48+
filename = "${path.module}/.coderv2/api_key"
49+
depends_on = [null_resource.api_key]
50+
}
51+
52+
resource "null_resource" "license" {
53+
provisioner "local-exec" {
54+
interpreter = ["/bin/bash", "-c"]
55+
command = <<EOF
56+
curl '${local.deployments.primary.url}/api/v2/licenses' \
57+
-H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
58+
--data-raw '{"license":"${var.coder_license}"}' \
59+
--insecure --silent --output /dev/null
60+
EOF
61+
}
62+
}
63+
64+
resource "null_resource" "europe_proxy_token" {
65+
provisioner "local-exec" {
66+
interpreter = ["/bin/bash", "-c"]
67+
command = <<EOF
68+
curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
69+
-H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
70+
--data-raw '{"name":"europe","display_name":"Europe","icon":"/emojis/1f950.png"}' \
71+
--insecure --silent \
72+
| jq -r .proxy_token > ${path.module}/.coderv2/europe_proxy_token
73+
EOF
74+
}
75+
76+
depends_on = [null_resource.license]
77+
}
78+
79+
data "local_file" "europe_proxy_token" {
80+
filename = "${path.module}/.coderv2/europe_proxy_token"
81+
depends_on = [null_resource.europe_proxy_token]
82+
}
83+
84+
resource "null_resource" "asia_proxy_token" {
85+
provisioner "local-exec" {
86+
interpreter = ["/bin/bash", "-c"]
87+
command = <<EOF
88+
curl '${local.deployments.primary.url}/api/v2/workspaceproxies' \
89+
-H "Coder-Session-Token: ${trimspace(data.local_file.api_key.content)}" \
90+
--data-raw '{"name":"asia","display_name":"Asia","icon":"/emojis/1f35b.png"}' \
91+
--insecure --silent \
92+
| jq -r .proxy_token > ${path.module}/.coderv2/asia_proxy_token
93+
EOF
94+
}
95+
96+
depends_on = [null_resource.license]
97+
}
98+
99+
data "local_file" "asia_proxy_token" {
100+
filename = "${path.module}/.coderv2/asia_proxy_token"
101+
depends_on = [null_resource.asia_proxy_token]
102+
}

‎scaletest/terraform/action/coder_templates.tf

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
resource "local_file" "kubernetes_template" {
2+
filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
3+
content = <<EOF
4+
terraform {
5+
required_providers {
6+
coder = {
7+
source = "coder/coder"
8+
version = "~> 0.23.0"
9+
}
10+
kubernetes = {
11+
source = "hashicorp/kubernetes"
12+
version = "~> 2.30"
13+
}
14+
}
15+
}
16+
17+
provider "coder" {}
18+
19+
provider "kubernetes" {
20+
config_path = null # always use host
21+
}
22+
23+
data "coder_workspace" "me" {}
24+
data "coder_workspace_owner" "me" {}
25+
26+
resource "coder_agent" "main" {
27+
os = "linux"
28+
arch = "amd64"
29+
}
30+
31+
resource "kubernetes_pod" "main" {
32+
count = data.coder_workspace.me.start_count
33+
metadata {
34+
name = "coder-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
35+
namespace = "${local.coder_namespace}"
36+
labels = {
37+
"app.kubernetes.io/name" = "coder-workspace"
38+
"app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
39+
}
40+
}
41+
spec {
42+
security_context {
43+
run_as_user = "1000"
44+
fs_group = "1000"
45+
}
46+
container {
47+
name = "dev"
48+
image = "${var.workspace_image}"
49+
image_pull_policy = "Always"
50+
command = ["sh", "-c", coder_agent.main.init_script]
51+
security_context {
52+
run_as_user = "1000"
53+
}
54+
env {
55+
name = "CODER_AGENT_TOKEN"
56+
value = coder_agent.main.token
57+
}
58+
resources {
59+
requests = {
60+
"cpu" = "${local.scenarios[var.scenario].workspaces.cpu_request}"
61+
"memory" = "${local.scenarios[var.scenario].workspaces.mem_request}"
62+
}
63+
limits = {
64+
"cpu" = "${local.scenarios[var.scenario].workspaces.cpu_limit}"
65+
"memory" = "${local.scenarios[var.scenario].workspaces.mem_limit}"
66+
}
67+
}
68+
}
69+
70+
affinity {
71+
node_affinity {
72+
required_during_scheduling_ignored_during_execution {
73+
node_selector_term {
74+
match_expressions {
75+
key = "cloud.google.com/gke-nodepool"
76+
operator = "In"
77+
values = ["${google_container_node_pool.node_pool["primary_workspaces"].name}","${google_container_node_pool.node_pool["europe_workspaces"].name}","${google_container_node_pool.node_pool["asia_workspaces"].name}"]
78+
}
79+
}
80+
}
81+
}
82+
}
83+
}
84+
}
85+
EOF
86+
}
87+
88+
resource "kubernetes_config_map" "template" {
89+
provider = kubernetes.primary
90+
91+
metadata {
92+
name = "coder-template"
93+
namespace = kubernetes_namespace.coder_primary.metadata.0.name
94+
}
95+
96+
data = {
97+
"main.tf" = local_file.kubernetes_template.content
98+
}
99+
}
100+
101+
resource "kubernetes_job" "push_template" {
102+
provider = kubernetes.primary
103+
104+
metadata {
105+
name = "${var.name}-push-template"
106+
namespace = kubernetes_namespace.coder_primary.metadata.0.name
107+
labels = {
108+
"app.kubernetes.io/name" = "${var.name}-push-template"
109+
}
110+
}
111+
spec {
112+
completions = 1
113+
template {
114+
metadata {}
115+
spec {
116+
affinity {
117+
node_affinity {
118+
required_during_scheduling_ignored_during_execution {
119+
node_selector_term {
120+
match_expressions {
121+
key = "cloud.google.com/gke-nodepool"
122+
operator = "In"
123+
values = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
124+
}
125+
}
126+
}
127+
}
128+
}
129+
container {
130+
name = "cli"
131+
image = "${var.coder_image_repo}:${var.coder_image_tag}"
132+
command = [
133+
"/opt/coder",
134+
"--verbose",
135+
"--url=${local.deployments.primary.url}",
136+
"--token=${trimspace(data.local_file.api_key.content)}",
137+
"templates",
138+
"push",
139+
"--directory=/home/coder/template",
140+
"--yes",
141+
"kubernetes"
142+
]
143+
volume_mount {
144+
name = "coder-template"
145+
mount_path = "/home/coder/template/main.tf"
146+
sub_path = "main.tf"
147+
}
148+
}
149+
volume {
150+
name = "coder-template"
151+
config_map {
152+
name = kubernetes_config_map.template.metadata.0.name
153+
}
154+
}
155+
restart_policy = "Never"
156+
}
157+
}
158+
}
159+
wait_for_completion = true
160+
}

0 commit comments

Comments
 (0)