test: attempt to make functional test startup more reliable

JohnVillalovos · JohnVillalovos · commit c829b263c9f6 · 2022-07-26T09:48:00.000-07:00
The functional tests have been erratic. Current theory is that we are starting the tests before the GitLab container is fully up and running. * Add checking of the Health Check[1] endpoints. * Add a 20 second delay after we believe it is up and running. * Increase timeout from 300 to 400 seconds [1] https://docs.gitlab.com/ee/user/admin_area/monitoring/health_check.html
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
@@ -6,11 +6,14 @@
 from subprocess import check_output
 
 import pytest
+import requests
 
 import gitlab
 import gitlab.base
 from tests.functional import helpers
 
+SLEEP_TIME = 20
+
 
 @pytest.fixture(scope="session")
 def fixture_dir(test_dir):
@@ -129,15 +132,30 @@ def check_is_alive():
     Return a healthcheck function fixture for the GitLab container spinup.
     """
 
-    def _check(container: str, start_time: float) -> bool:
+    def _check(
+        *, container: str, start_time: float, docker_ip: str, docker_port: int
+    ) -> bool:
         setup_time = time.perf_counter() - start_time
         minutes, seconds = int(setup_time / 60), int(setup_time % 60)
         logging.info(
             f"Checking if GitLab container is up. "
             f"Have been checking for {minutes} minute(s), {seconds} seconds ..."
         )
         logs = ["docker", "logs", container]
-        return "gitlab Reconfigured!" in check_output(logs).decode()
+        if "gitlab Reconfigured!" not in check_output(logs).decode():
+            return False
+        logging.debug("GitLab has finished reconfiguring.")
+        gitlab_url = f"http://{docker_ip}:{docker_port}"
+        for check in ("health", "readiness", "liveness"):
+            url = f"{gitlab_url}/-/{check}"
+            logging.debug(f"Checking {check!r} endpoint at: {url}")
+            result = requests.get(url)
+            if result.status_code != 200:
+                logging.debug(f"{check!r} check did not return 200: {result}")
+                return False
+        logging.debug(f"Sleeping for {SLEEP_TIME}")
+        time.sleep(SLEEP_TIME)
+        return True
 
     return _check
 
@@ -174,9 +192,14 @@ def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_
     start_time = time.perf_counter()
     logging.info("Waiting for GitLab container to become ready.")
     docker_services.wait_until_responsive(
-        timeout=300,
+        timeout=400,
         pause=10,
-        check=lambda: check_is_alive("gitlab-test", start_time=start_time),
+        check=lambda: check_is_alive(
+            container="gitlab-test",
+            start_time=start_time,
+            docker_ip=docker_ip,
+            docker_port=port,
+        ),
     )
     setup_time = time.perf_counter() - start_time
     minutes, seconds = int(setup_time / 60), int(setup_time % 60)
@@ -208,6 +231,7 @@ def gl(gitlab_config):
     logging.info("Instantiating python-gitlab gitlab.Gitlab instance")
     instance = gitlab.Gitlab.from_config("local", [gitlab_config])
 
+    logging.info("Reset GitLab")
     reset_gitlab(instance)
 
     return instance
diff --git a/tests/functional/fixtures/docker-compose.yml b/tests/functional/fixtures/docker-compose.yml
@@ -32,6 +32,7 @@ services:
         grafana['enable'] = false
         letsencrypt['enable'] = false
         gitlab_rails['initial_license_file'] = '/python-gitlab-ci.gitlab-license'
+        gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0']
     entrypoint:
       - /bin/sh
       - -c
