Skip to content

Commit 3a86dfc

Browse files
Jan-MJan-M
authored
End 2 End tests speedup (zalando#1180)
* Improving end 2 end tests, especially speed of execution and error, by implementing proper eventual asserts and timeouts. * Add documentation for running individual tests * Fixed String encoding in Patorni state check and error case * Printing config as multi log line entity, makes it readable and grepable on startup * Cosmetic changes to logs. Removed quotes from diff. Move all object diffs to text diff. Enabled padding for log level. * Mount script with tools for easy logaccess and watching objects. * Set proper update strategy for Postgres operator deployment. * Move long running test to end. Move pooler test to new functions. * Remove quote from valid K8s identifiers.
1 parent 7730ecf commit 3a86dfc

24 files changed

+1326
-601
lines changed

‎Makefile

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,16 @@ PKG := `go list ./... | grep -v /vendor/`
2424

2525
ifeq ($(DEBUG),1)
2626
DOCKERFILE = DebugDockerfile
27-
DEBUG_POSTFIX := -debug
27+
DEBUG_POSTFIX := -debug-$(shell date hhmmss)
2828
BUILD_FLAGS += -gcflags "-N -l"
2929
else
3030
DOCKERFILE = Dockerfile
3131
endif
3232

33+
ifeq ($(FRESH),1)
34+
DEBUG_FRESH=$(shell date +"%H-%M-%S")
35+
endif
36+
3337
ifdef CDP_PULL_REQUEST_NUMBER
3438
CDP_TAG := -${CDP_BUILD_VERSION}
3539
endif
@@ -66,7 +70,7 @@ docker: ${DOCKERDIR}/${DOCKERFILE} docker-context
6670
echo "Version ${VERSION}"
6771
echo "CDP tag ${CDP_TAG}"
6872
echo "git describe $(shell git describe --tags --always --dirty)"
69-
cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" .
73+
cd "${DOCKERDIR}" && docker build --rm -t "$(IMAGE):$(TAG)$(CDP_TAG)$(DEBUG_FRESH)$(DEBUG_POSTFIX)" -f "${DOCKERFILE}" .
7074

7175
indocker-race:
7276
docker run --rm -v "${GOPATH}":"${GOPATH}" -e GOPATH="${GOPATH}" -e RACE=1 -w ${PWD} golang:1.8.1 bash -c "make linux"
@@ -97,4 +101,4 @@ test:
97101
GO111MODULE=on go test ./...
98102

99103
e2e: docker # build operator image to be tested
100-
cd e2e; make e2etest
104+
cd e2e; make e2etest

‎e2e/Dockerfile

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,15 @@ RUN apt-get update \
1414
python3-setuptools \
1515
python3-pip \
1616
curl \
17+
vim \
1718
&& pip3 install --no-cache-dir -r requirements.txt \
1819
&& curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl \
1920
&& chmod +x ./kubectl \
2021
&& mv ./kubectl /usr/local/bin/kubectl \
2122
&& apt-get clean \
2223
&& rm -rf /var/lib/apt/lists/*
2324

24-
ENTRYPOINT ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]
25+
# working line
26+
# python3 -m unittest discover -v --failfast -k test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade --start-directory tests
27+
ENTRYPOINT ["python3", "-m", "unittest"]
28+
CMD ["discover","-v","--failfast","--start-directory","/tests"]

‎e2e/README.md

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ Docker.
1212
Docker
1313
Go
1414

15+
# Notice
16+
17+
The `manifest` folder in e2e tests folder is not commited to git, it comes from `/manifests`
18+
1519
## Build test runner
1620

1721
In the directory of the cloned Postgres Operator repository change to the e2e
@@ -35,6 +39,46 @@ In the e2e folder you can invoke tests either with `make test` or with:
3539
To run both the build and test step you can invoke `make e2e` from the parent
3640
directory.
3741

42+
To run the end 2 end test and keep the kind state execute:
43+
```bash
44+
NOCLEANUP=True ./run.sh
45+
```
46+
47+
## Run indidual test
48+
49+
After having executed a normal E2E run with `NOCLEANUP=True` Kind still continues to run, allowing you subsequent test runs.
50+
51+
To run an individual test, run the following command in the `e2e` directory
52+
53+
```bash
54+
NOCLEANUP=True ./run.sh main tests.test_e2e.EndToEndTestCase.test_lazy_spilo_upgrade
55+
```
56+
57+
## Inspecting Kind
58+
59+
If you want to inspect Kind/Kubernetes cluster, use the following script to exec into the K8s setup and then use `kubectl`
60+
61+
```bash
62+
./exec_into_env.sh
63+
64+
# use kube ctl
65+
kubectl get pods
66+
67+
# watch relevant objects
68+
./scripts/watch_objects.sh
69+
70+
# get operator logs
71+
./scripts/get_logs.sh
72+
```
73+
74+
## Cleaning up Kind
75+
76+
To cleanup kind and start fresh
77+
78+
```bash
79+
e2e/run.sh cleanup
80+
```
81+
3882
## Covered use cases
3983

4084
The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):

‎e2e/exec_into_env.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
export cluster_name="postgres-operator-e2e-tests"
4+
export kubeconfig_path="/tmp/kind-config-${cluster_name}"
5+
export operator_image="registry.opensource.zalan.do/acid/postgres-operator:latest"
6+
export e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3"
7+
8+
docker run -it --entrypoint /bin/bash --network=host -e "TERM=xterm-256color" \
9+
--mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \
10+
--mount type=bind,source="$(readlink -f manifests)",target=/manifests \
11+
--mount type=bind,source="$(readlink -f tests)",target=/tests \
12+
--mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \
13+
--mount type=bind,source="$(readlink -f scripts)",target=/scripts \
14+
-e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}"

‎e2e/run.sh

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ IFS=$'\n\t'
99
readonly cluster_name="postgres-operator-e2e-tests"
1010
readonly kubeconfig_path="/tmp/kind-config-${cluster_name}"
1111
readonly spilo_image="registry.opensource.zalan.do/acid/spilo-12:1.6-p5"
12+
readonly e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:0.3"
13+
14+
export GOPATH=${GOPATH-~/go}
15+
export PATH=${GOPATH}/bin:$PATH
1216

1317
echo "Clustername: ${cluster_name}"
1418
echo "Kubeconfig path: ${kubeconfig_path}"
@@ -19,12 +23,7 @@ function pull_images(){
1923
then
2024
docker pull registry.opensource.zalan.do/acid/postgres-operator:latest
2125
fi
22-
2326
operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1)
24-
25-
# this image does not contain the tests; a container mounts them from a local "./tests" dir at start time
26-
e2e_test_runner_image="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests-runner:latest"
27-
docker pull ${e2e_test_runner_image}
2827
}
2928

3029
function start_kind(){
@@ -36,12 +35,17 @@ function start_kind(){
3635
fi
3736

3837
export KUBECONFIG="${kubeconfig_path}"
39-
kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml
40-
kind load docker-image "${operator_image}" --name ${cluster_name}
38+
kind create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml
4139
docker pull "${spilo_image}"
4240
kind load docker-image "${spilo_image}" --name ${cluster_name}
4341
}
4442

43+
function load_operator_image() {
44+
echo "Loading operator image"
45+
export KUBECONFIG="${kubeconfig_path}"
46+
kind load docker-image "${operator_image}" --name ${cluster_name}
47+
}
48+
4549
function set_kind_api_server_ip(){
4650
echo "Setting up kind API server ip"
4751
# use the actual kubeconfig to connect to the 'kind' API server
@@ -52,35 +56,36 @@ function set_kind_api_server_ip(){
5256
}
5357

5458
function run_tests(){
55-
echo "Running tests..."
56-
59+
echo "Running tests... image: ${e2e_test_runner_image}"
5760
# tests modify files in ./manifests, so we mount a copy of this directory done by the e2e Makefile
5861

5962
docker run --rm --network=host -e "TERM=xterm-256color" \
6063
--mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config \
6164
--mount type=bind,source="$(readlink -f manifests)",target=/manifests \
6265
--mount type=bind,source="$(readlink -f tests)",target=/tests \
6366
--mount type=bind,source="$(readlink -f exec.sh)",target=/exec.sh \
64-
-e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}"
65-
67+
--mount type=bind,source="$(readlink -f scripts)",target=/scripts \
68+
-e OPERATOR_IMAGE="${operator_image}" "${e2e_test_runner_image}" ${E2E_TEST_CASE-} $@
6669
}
6770

68-
function clean_up(){
71+
function cleanup(){
6972
echo "Executing cleanup"
7073
unset KUBECONFIG
7174
kind delete cluster --name ${cluster_name}
7275
rm -rf ${kubeconfig_path}
7376
}
7477

7578
function main(){
76-
77-
trap "clean_up" QUIT TERM EXIT
78-
79-
time pull_images
80-
time start_kind
81-
time set_kind_api_server_ip
82-
run_tests
79+
echo "Entering main function..."
80+
[[ -z ${NOCLEANUP-} ]] && trap "cleanup" QUIT TERM EXIT
81+
pull_images
82+
[[ ! -f ${kubeconfig_path} ]] && start_kind
83+
load_operator_image
84+
set_kind_api_server_ip
85+
86+
shift
87+
run_tests $@
8388
exit 0
8489
}
8590

86-
"$@"
91+
"$1" $@

‎e2e/scripts/cleanup.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
kubectl delete postgresql acid-minimal-cluster
3+
kubectl delete deployments -l application=db-connection-pooler,cluster-name=acid-minimal-cluster
4+
kubectl delete statefulsets -l application=spilo,cluster-name=acid-minimal-cluster
5+
kubectl delete services -l application=spilo,cluster-name=acid-minimal-cluster
6+
kubectl delete configmap postgres-operator
7+
kubectl delete deployment postgres-operator

‎e2e/scripts/get_logs.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/bash
2+
kubectl logs $(kubectl get pods -l name=postgres-operator --field-selector status.phase=Running -o jsonpath='{.items..metadata.name}')

‎e2e/scripts/watch_objects.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
watch -c "
4+
kubectl get postgresql
5+
echo
6+
echo -n 'Rolling upgrade pending: '
7+
kubectl get statefulset -o jsonpath='{.items..metadata.annotations.zalando-postgres-operator-rolling-update-required}'
8+
echo
9+
echo
10+
kubectl get pods -o wide
11+
echo
12+
kubectl get statefulsets
13+
echo
14+
kubectl get deployments
15+
echo
16+
kubectl get pods -l name=postgres-operator -o jsonpath='{.items..metadata.annotations.step}'
17+
echo
18+
kubectl get pods -l application=spilo -o jsonpath='{.items..spec.containers..image}'
19+
"

0 commit comments

Comments
 (0)