wip: make reset_gitlab() better

JohnVillalovos · JohnVillalovos · commit ea4c50df6bfc · 2021-12-28T21:41:23.000-08:00
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
@@ -1,3 +1,4 @@
+import logging
 import tempfile
 import time
 import uuid
@@ -10,54 +11,119 @@
 import gitlab.base
 
 SLEEP_INTERVAL = 0.1
-TIMEOUT = 60  # seconds before timeout will occur
+TIMEOUT = 3 * 60  # seconds before timeout will occur
 
 
 @pytest.fixture(scope="session")
 def fixture_dir(test_dir):
     return test_dir / "functional" / "fixtures"
 
 
-def reset_gitlab(gl):
-    # previously tools/reset_gitlab.py
+def reset_gitlab(gl: gitlab.Gitlab) -> None:
+    # Mark our resources for deletion. It takes time for them to actually be deleted
+    # though.
+    _reset_gitlab_delete_resources(gl=gl)
+
+    # Wait for all resources to be deleted
+    _reset_gitlab_wait_deletion_finish(gl=gl)
+
+
+def _reset_gitlab_delete_resources(gl: gitlab.Gitlab) -> None:
+    """Mark for deletion, resources (such as projects, groups, users) that shouldn't
+    exist. Once marked they will still take time to be deleted."""
     for project in gl.projects.list():
+        logging.info(f"Mark for deletion project: {project.name!r}")
         for deploy_token in project.deploytokens.list():
+            logging.info(
+                f"Mark for deletion token: {deploy_token.name!r} in "
+                f"project: {project.name!r}"
+            )
             deploy_token.delete()
         project.delete()
     for group in gl.groups.list():
+        logging.info(f"Mark for deletion group: {group.name!r}")
         for deploy_token in group.deploytokens.list():
+            logging.info(
+                f"Mark for deletion token: {deploy_token.name!r} in "
+                f"group: {group.name!r}"
+            )
             deploy_token.delete()
         group.delete()
     for variable in gl.variables.list():
+        logging.info(f"Mark for deletion variable: {variable.name!r}")
         variable.delete()
     for user in gl.users.list():
         if user.username != "root":
+            logging.info(f"Mark for deletion user: {user.username!r}")
             user.delete(hard_delete=True)
 
+
+def _reset_gitlab_wait_deletion_finish(gl: gitlab.Gitlab) -> None:
+    """Wait for all of our resources to be deleted.
+
+    If anything exists then mark it again for deletion in case initial call to delete
+    didn't work, which has been seen :("""
+
     max_iterations = int(TIMEOUT / SLEEP_INTERVAL)
 
     # Ensure everything has been reset
     start_time = time.perf_counter()
 
     def wait_for_maximum_list_length(
-        rest_manager: gitlab.base.RESTManager, description: str, max_length: int = 0
+        rest_manager: gitlab.base.RESTManager,
+        description: str,
+        max_length: int = 0,
+        should_delete_func=lambda x: True,
+        delete_kwargs={},
     ) -> None:
         """Wait for the list() length to be no greater than expected maximum or fail
         test if timeout is exceeded"""
-        for _ in range(max_iterations):
-            if len(rest_manager.list()) <= max_length:
+        for count in range(max_iterations):
+            items = rest_manager.list()
+            logging.info(
+                f"Iteration: {count}: items in {description}: {[x.name for x in items]}"
+            )
+            for item in items:
+                if should_delete_func(item):
+                    logging.info(
+                        f"Marking again for deletion {description}: {item.name!r}"
+                    )
+                    try:
+                        item.delete(**delete_kwargs)
+                    except gitlab.exceptions.GitlabDeleteError as exc:
+                        logging.info(
+                            f"Already marked for deletion: {item.name!r} {exc}"
+                        )
+            if len(items) <= max_length:
                 break
             time.sleep(SLEEP_INTERVAL)
-        assert len(rest_manager.list()) <= max_length, (
+        items = rest_manager.list()
+        elapsed_time = time.perf_counter() - start_time
+        if len(items) > max_length:
+            logging.error(
+                f"Too many items still remaining and timeout exceeded: {elapsed_time}"
+            )
+        assert len(items) <= max_length, (
             f"Did not delete required items for {description}. "
-            f"Elapsed_time: {time.perf_counter() - start_time}"
+            f"Elapsed_time: {time.perf_counter() - start_time}\n"
+            f"items: {[str(x) for x in items]!r}"
         )
 
     wait_for_maximum_list_length(rest_manager=gl.projects, description="projects")
     wait_for_maximum_list_length(rest_manager=gl.groups, description="groups")
     wait_for_maximum_list_length(rest_manager=gl.variables, description="variables")
+
+    def should_delete_user(user):
+        if user.username == "root":
+            return False
+        return True
+
     wait_for_maximum_list_length(
-        rest_manager=gl.users, description="users", max_length=1
+        rest_manager=gl.users,
+        description="users",
+        max_length=1,
+        should_delete_func=should_delete_user,
+        delete_kwargs={"hard_delete": True},
     )
 
 
@@ -144,7 +210,7 @@ def wait_for_sidekiq(gl):
     """
 
     def _wait(timeout=30, step=0.5):
-        for _ in range(timeout):
+        for count in range(timeout):
             time.sleep(step)
             busy = False
             processes = gl.sidekiq.process_metrics()["processes"]
@@ -153,6 +219,7 @@ def _wait(timeout=30, step=0.5):
                     busy = True
             if not busy:
                 return True
+            logging.info(f"sidekiq busy {count} of {timeout}")
         return False
 
     return _wait
@@ -188,8 +255,10 @@ def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_
 def gl(gitlab_config):
     """Helper instance to make fixtures and asserts directly via the API."""
 
+    logging.info("Create python-gitlab gitlab.Gitlab object")
     instance = gitlab.Gitlab.from_config("local", [gitlab_config])
-    reset_gitlab(instance)
+
+    reset_gitlab(gl=instance)
 
     return instance
 
