diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0cbe44a28..f8ee64bbe 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -52,6 +52,8 @@ jobs:
           python-version: ${{ matrix.python.version }}
       - name: Install dependencies
         run: pip3 install tox pytest-github-actions-annotate-failures
+      - name: Setup dummy config
+        run: tests/ci_tools/setup-dummy-config.sh
       - name: Run tests
         env:
           TOXENV: ${{ matrix.python.toxenv }}
diff --git a/tests/ci_tools/setup-dummy-config.sh b/tests/ci_tools/setup-dummy-config.sh
new file mode 100755
index 000000000..4a316182a
--- /dev/null
+++ b/tests/ci_tools/setup-dummy-config.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -u
+
+if [[ -z "${CI:-}" || -z "${GITHUB_ACTIONS:-}" ]]; then
+    echo "ERROR: Not running in the GitHub CI."
+    exit 2
+fi
+
+CONFIG_FILE=~/.python-gitlab.cfg
+if [[ -e "${CONFIG_FILE}" ]]; then
+    echo "ERROR: Config file already exists: ${CONFIG_FILE}"
+    echo "Saved you from destroying your config"
+    exit 2
+fi
+
+cat <<EOF >"${CONFIG_FILE}"
+[global]
+default = gitlab
+ssl_verify = true
+timeout = 5
+api_version = 4
+
+[gitlab]
+url = https://gitlab.com/
+private_token = not-a-valid-token
+EOF
+
+echo "Setup config at: ${CONFIG_FILE}"