Skip to content

Commit fae895c

Browse files
cbandycbandy
authored
Move pgbackrest-restore test to Kyverno Chainsaw (#4228)
* Move pgbackrest-restore test to Kyverno Chainsaw This test has a number of scripts and jobs that pass and share data. Chainsaw's bindings and templates are a nice way to break this test up, and its "catch" operations provide good context when a step fails. Tested with Kyverno Chainsaw 0.2.12 See: https://kyverno.github.io/chainsaw/main This helped me reproduce a race in "pg_ctl start" during Postgres recovery. Issue: PGO-1945
1 parent c7842e7 commit fae895c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+903
-680
lines changed

‎.github/workflows/test.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ jobs:
8686
path: envtest-existing.coverage.gz
8787
retention-days: 1
8888

89-
kuttl-k3d:
89+
e2e-k3d:
9090
runs-on: ubuntu-24.04
9191
needs: [go-test]
9292
strategy:
@@ -144,10 +144,12 @@ jobs:
144144
--env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.2-0' \
145145
--env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
146146
--name 'postgres-operator' localhost/postgres-operator
147-
- name: Install kuttl
148-
run: |
149-
curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
150-
chmod +x /usr/local/bin/kubectl-kuttl
147+
148+
- run: |
149+
make check-chainsaw && exit
150+
failed=$?
151+
echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
152+
exit $failed
151153
152154
- run: make generate-kuttl
153155
env:
@@ -161,8 +163,6 @@ jobs:
161163
failed=$?
162164
echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
163165
exit $failed
164-
env:
165-
KUTTL: kubectl-kuttl
166166
167167
- name: Stop PGO
168168
run: docker stop 'postgres-operator' || true

‎Makefile

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ GO_TEST ?= $(GO) test
1212
CONTROLLER ?= $(GO) tool sigs.k8s.io/controller-tools/cmd/controller-gen
1313

1414
# Run tests using the latest tools.
15+
CHAINSAW ?= $(GO) run github.com/kyverno/chainsaw@latest
16+
CHAINSAW_TEST ?= $(CHAINSAW) test
1517
ENVTEST ?= $(GO) run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
1618
KUTTL ?= $(GO) run github.com/kudobuilder/kuttl/cmd/kubectl-kuttl@latest
1719
KUTTL_TEST ?= $(KUTTL) test
@@ -170,6 +172,17 @@ check-envtest-existing: createnamespaces
170172
$(GO_TEST) -count=1 -cover -p=1 ./...
171173
kubectl delete -k ./config/dev
172174

175+
# Expects operator to be running
176+
#
177+
# Chainsaw runs with a single kubectl context named "chainsaw".
178+
# If you experience `cluster "minikube" does not exist`, try `MINIKUBE_PROFILE=chainsaw`.
179+
#
180+
# https://kyverno.github.io/chainsaw/latest/operations/script#kubeconfig
181+
#
182+
.PHONY: check-chainsaw
183+
check-chainsaw:
184+
$(CHAINSAW_TEST) --config testing/chainsaw/e2e/config.yaml --values testing/chainsaw/e2e/values.yaml testing/chainsaw/e2e
185+
173186
# Expects operator to be running
174187
#
175188
# KUTTL runs with a single kubectl context named "cluster".

‎testing/chainsaw/e2e/config.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: chainsaw.kyverno.io/v1alpha2
2+
kind: Configuration
3+
metadata:
4+
name: end-to-end
5+
spec:
6+
namespace:
7+
template:
8+
metadata:
9+
labels: { postgres-operator-test: chainsaw }
10+
timeouts:
11+
assert: 3m
12+
cleanup: 3m

‎testing/chainsaw/e2e/pgbackrest-restore/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# pgbackrest-restore
2+
3+
This [chainsaw](https://github.com/kyverno/chainsaw) suite tests that CPK can clone and restore through pgBackRest backups.
4+
5+
This md page is meant as a placeholder for further documentation as necessary of this particular test.

‎testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
apiVersion: chainsaw.kyverno.io/v1alpha1
2+
kind: Test
3+
metadata:
4+
name: pgbackrest-restore
5+
labels:
6+
pgbackrest: ''
7+
spec:
8+
failFast: true
9+
bindings:
10+
- name: postgres
11+
value:
12+
version: (to_number(as_string($values.versions.postgres)))
13+
14+
- name: psql
15+
value:
16+
image: ($values.images.psql)
17+
connect: { name: PGCONNECT_TIMEOUT, value: '5' }
18+
19+
- name: volume
20+
value: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } }
21+
22+
steps:
23+
- name: 'Create Cluster with replica, tablespace'
24+
use:
25+
template: 'templates/create-cluster.yaml'
26+
27+
- name: 'Create Data'
28+
use:
29+
template: 'templates/psql-data.yaml'
30+
with:
31+
bindings:
32+
- name: target
33+
value: original
34+
- name: job
35+
value: original-data
36+
- name: command
37+
value: |
38+
CREATE SCHEMA IF NOT EXISTS "original";
39+
CREATE TABLE important (data) AS VALUES ('treasure');
40+
CREATE TABLE cows (name) TABLESPACE barn AS VALUES ('nellie');
41+
42+
- name: 'Create Backup #1'
43+
use:
44+
template: 'templates/create-backup.yaml'
45+
with:
46+
bindings:
47+
- name: annotation
48+
value: one
49+
50+
- name: 'Clone Cluster #1'
51+
skipDelete: true
52+
use:
53+
template: 'templates/clone-cluster.yaml'
54+
with:
55+
bindings:
56+
- name: name
57+
value: clone-one
58+
59+
- name: 'Verify Data on Clone #1'
60+
use:
61+
template: 'templates/psql-data.yaml'
62+
with:
63+
bindings:
64+
- name: target
65+
value: clone-one
66+
- name: job
67+
value: clone-one-data
68+
- name: command
69+
value: |
70+
DO $$$$
71+
DECLARE
72+
restored jsonb;
73+
BEGIN
74+
SELECT jsonb_agg(important) INTO restored FROM important;
75+
ASSERT restored = '[{"data":"treasure"}]', format('got %L', restored);
76+
SELECT jsonb_agg(cows) INTO restored FROM cows;
77+
ASSERT restored = '[{"name":"nellie"}]', format('got %L', restored);
78+
END $$$$;
79+
80+
- name: 'Delete Cluster #1'
81+
description: >
82+
Delete this clone in the background to free up resources
83+
try:
84+
- delete:
85+
deletionPropagationPolicy: Background
86+
expect: [{ check: { (`true`): true } }]
87+
ref:
88+
apiVersion: postgres-operator.crunchydata.com/v1beta1
89+
kind: PostgresCluster
90+
name: clone-one
91+
92+
- name: 'Restart Cluster'
93+
description: >
94+
Sets a timestamp and restarts the cluster, using the timestamp for comparison
95+
use:
96+
template: 'templates/restart-cluster.yaml'
97+
98+
- name: 'Update Data'
99+
use:
100+
template: 'templates/psql-data.yaml'
101+
with:
102+
bindings:
103+
- name: target
104+
value: original
105+
- name: job
106+
value: original-more-data
107+
- name: command
108+
value: INSERT INTO important (data) VALUES ('water'), ('socks');
109+
110+
- name: 'Verify WAL backup'
111+
use:
112+
template: 'templates/verify-backup.yaml'
113+
114+
- name: 'Create Backup #2'
115+
use:
116+
template: 'templates/create-backup.yaml'
117+
with:
118+
bindings:
119+
- name: annotation
120+
value: two
121+
122+
- name: 'Clone Cluster #2'
123+
skipDelete: true
124+
use:
125+
template: 'templates/clone-cluster.yaml'
126+
with:
127+
bindings:
128+
- name: name
129+
value: clone-two
130+
131+
- name: 'Verify Data on Clone #2'
132+
use:
133+
template: 'templates/psql-data.yaml'
134+
with:
135+
bindings:
136+
- name: target
137+
value: clone-two
138+
- name: job
139+
value: clone-two-data
140+
- name: command
141+
value: |
142+
DO $$$$
143+
DECLARE
144+
restored jsonb;
145+
BEGIN
146+
SELECT jsonb_agg(important) INTO restored FROM important;
147+
ASSERT restored = '[
148+
{"data":"treasure"}, {"data":"water"}, {"data":"socks"}
149+
]', format('got %L', restored);
150+
END $$$$;
151+
152+
- name: 'Delete Cluster #2'
153+
description: >
154+
Delete this clone in the background to free up resources
155+
try:
156+
- delete:
157+
deletionPropagationPolicy: Background
158+
expect: [{ check: { (`true`): true } }]
159+
ref:
160+
apiVersion: postgres-operator.crunchydata.com/v1beta1
161+
kind: PostgresCluster
162+
name: clone-two
163+
164+
- name: 'Lose Data'
165+
description: >
166+
Drop data and ensure that the data is dropped from the replica as well
167+
use:
168+
template: 'templates/lose-data.yaml'
169+
170+
- name: 'Point-In-Time Restore'
171+
use:
172+
template: 'templates/point-in-time-restore.yaml'
173+
174+
- name: 'Verify Primary'
175+
description: >
176+
Confirm that data was restored to the point-in-time and the cluster is healthy
177+
use:
178+
template: 'templates/psql-data.yaml'
179+
with:
180+
bindings:
181+
- name: target
182+
value: original
183+
- name: job
184+
value: original-pitr-primary
185+
- name: command
186+
value: |
187+
DO $$$$
188+
DECLARE
189+
restored jsonb;
190+
BEGIN
191+
SELECT jsonb_agg(important) INTO restored FROM important;
192+
ASSERT restored = '[
193+
{"data":"treasure"}, {"data":"water"}, {"data":"socks"}
194+
]', format('got %L', restored);
195+
END $$$$;
196+
197+
- name: 'Confirm Replica'
198+
description: >
199+
Verify that the data has streamed and is streaming to the replica
200+
use:
201+
template: 'templates/verify-replica.yaml'

‎testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
apiVersion: chainsaw.kyverno.io/v1alpha1
2+
kind: StepTemplate
3+
metadata:
4+
name: clone-cluster
5+
spec:
6+
bindings:
7+
- name: name
8+
value: 'The name of the new PostgresCluster'
9+
10+
try:
11+
-
12+
description: >
13+
Clone the cluster using a pgBackRest restore
14+
apply:
15+
resource:
16+
apiVersion: postgres-operator.crunchydata.com/v1beta1
17+
kind: PostgresCluster
18+
metadata:
19+
name: ($name)
20+
spec:
21+
dataSource:
22+
postgresCluster:
23+
clusterName: original
24+
repoName: repo1
25+
postgresVersion: ($postgres.version)
26+
instances:
27+
- dataVolumeClaimSpec: ($volume)
28+
tablespaceVolumes:
29+
- { name: barn, dataVolumeClaimSpec: ($volume) }
30+
backups:
31+
pgbackrest:
32+
repos:
33+
- name: repo1
34+
volume:
35+
volumeClaimSpec: ($volume)
36+
37+
-
38+
description: >
39+
Wait for the cluster to come online
40+
assert:
41+
resource:
42+
apiVersion: postgres-operator.crunchydata.com/v1beta1
43+
kind: PostgresCluster
44+
metadata:
45+
name: ($name)
46+
status:
47+
instances:
48+
- name: '00'
49+
replicas: 1
50+
readyReplicas: 1
51+
updatedReplicas: 1

‎testing/chainsaw/e2e/pgbackrest-restore/templates/create-backup.yaml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
apiVersion: chainsaw.kyverno.io/v1alpha1
2+
kind: StepTemplate
3+
metadata:
4+
name: create-backup
5+
spec:
6+
bindings:
7+
- name: annotation
8+
value: 'The annotation to kick off a backup'
9+
try:
10+
-
11+
description: >
12+
Annotate the cluster to trigger a backup
13+
patch:
14+
resource:
15+
apiVersion: postgres-operator.crunchydata.com/v1beta1
16+
kind: PostgresCluster
17+
metadata:
18+
name: original
19+
annotations:
20+
postgres-operator.crunchydata.com/pgbackrest-backup: ($annotation)
21+
22+
-
23+
description: >
24+
Wait for the backup to complete
25+
assert:
26+
resource:
27+
apiVersion: batch/v1
28+
kind: Job
29+
metadata:
30+
annotations:
31+
postgres-operator.crunchydata.com/pgbackrest-backup: ($annotation)
32+
labels:
33+
postgres-operator.crunchydata.com/cluster: original
34+
postgres-operator.crunchydata.com/pgbackrest-backup: manual
35+
status:
36+
succeeded: 1

0 commit comments

Comments
 (0)