feat(ci): improve functionnal tests

jouve · jouve · commit eefceace2c20 · 2019-10-06T16:39:33.000+02:00
diff --git a/tools/build_test_env.sh b/tools/build_test_env.sh
@@ -25,11 +25,13 @@ error() { log "ERROR: $@" >&2; }
 fatal() { error "$@"; exit 1; }
 try() { "$@" || fatal "'$@' failed"; }
 
+REUSE_CONTAINER=
 NOVENV=
 PY_VER=3
 API_VER=4
-while getopts :np:a: opt "$@"; do
+while getopts :knp:a: opt "$@"; do
     case $opt in
+        k) REUSE_CONTAINER=1;;
         n) NOVENV=1;;
         p) PY_VER=$OPTARG;;
         a) API_VER=$OPTARG;;
@@ -67,18 +69,42 @@ cleanup() {
     command -v deactivate >/dev/null 2>&1 && deactivate || true
     log "Deleting python virtualenv..."
     rm -rf "$VENV"
-    log "Stopping gitlab-test docker container..."
-    docker rm -f gitlab-test >/dev/null
+    if [ -z "$REUSE_CONTAINER" ]; then
+        log "Stopping gitlab-test docker container..."
+        docker rm -f gitlab-test >/dev/null
+    fi
     log "Done."
 }
 [ -z "${BUILD_TEST_ENV_AUTO_CLEANUP+set}" ] || {
     trap cleanup EXIT
     trap 'exit 1' HUP INT TERM
 }
 
-try docker pull registry.gitlab.com/python-gitlab/python-gitlab:test >/dev/null
-try docker run --name gitlab-test --detach --publish 8080:80 \
-    --publish 2222:22 registry.gitlab.com/python-gitlab/python-gitlab:test >/dev/null
+if [ -z "$REUSE_CONTAINER" ] || ! docker top gitlab-test >/dev/null 2>&1; then
+    GITLAB_OMNIBUS_CONFIG="external_url 'http://gitlab.test'
+gitlab_rails['initial_root_password'] = '5iveL!fe'
+gitlab_rails['initial_shared_runners_registration_token'] = 'sTPNtWLEuSrHzoHP8oCU'
+registry['enable'] = false
+nginx['redirect_http_to_https'] = false
+nginx['listen_port'] = 80
+nginx['listen_https'] = false
+pages_external_url 'http://pages.gitlab.lxd'
+gitlab_pages['enable'] = true
+gitlab_pages['inplace_chroot'] = true
+prometheus['enable'] = false
+alertmanager['enable'] = false
+node_exporter['enable'] = false
+redis_exporter['enable'] = false
+postgres_exporter['enable'] = false
+pgbouncer_exporter['enable'] = false
+gitlab_exporter['enable'] = false
+grafana['enable'] = false
+letsencrypt['enable'] = false
+"
+    try docker run --name gitlab-test --detach --publish 8080:80 \
+        --publish 2222:22 --env "GITLAB_OMNIBUS_CONFIG=$GITLAB_OMNIBUS_CONFIG" \
+        gitlab/gitlab-ce:latest >/dev/null
+fi
 
 LOGIN='root'
 PASSWORD='5iveL!fe'
@@ -106,7 +132,7 @@ if [ -z "$NOVENV" ]; then
     try pip install -e .
 
     # to run generate_token.py
-    pip install bs4 lxml
+    pip install requests-html
 fi
 
 log "Waiting for gitlab to come online... "
@@ -115,12 +141,20 @@ while :; do
     sleep 1
     docker top gitlab-test >/dev/null 2>&1 || fatal "docker failed to start"
     sleep 4
-    curl -s http://localhost:8080/users/sign_in 2>/dev/null \
-        | grep -q "GitLab Community Edition" && break
+    # last command started by the container is "gitlab-ctl tail"
+    docker exec gitlab-test pgrep -f 'gitlab-ctl tail' &>/dev/null \
+    && docker exec gitlab-test curl http://localhost/-/health 2>/dev/null \
+        | grep -q 'GitLab OK' \
+    && curl -s http://localhost:8080/users/sign_in 2>/dev/null \
+        | grep -q "GitLab Community Edition" \
+    && break
     I=$((I+5))
     [ "$I" -lt 180 ] || fatal "timed out"
 done
 
+log "Pausing to give GitLab some time to finish starting up..."
+sleep 200
+
 # Get the token
 TOKEN=$($(dirname $0)/generate_token.py)
 
@@ -138,7 +172,8 @@ EOF
 log "Config file content ($CONFIG):"
 log <$CONFIG
 
-log "Pausing to give GitLab some time to finish starting up..."
-sleep 200
-
+if [ ! -z "$REUSE_CONTAINER" ]; then
+    echo reset gitlab
+    $(dirname $0)/reset_gitlab.py
+fi
 log "Test environment initialized."
diff --git a/tools/generate_token.py b/tools/generate_token.py
@@ -1,64 +1,50 @@
 #!/usr/bin/env python
 
-import sys
-
-try:
-    from urllib.parse import urljoin
-except ImportError:
-    from urlparse import urljoin
-
-from bs4 import BeautifulSoup
-import requests
-
-endpoint = "http://localhost:8080"
-root_route = urljoin(endpoint, "/")
-sign_in_route = urljoin(endpoint, "/users/sign_in")
-pat_route = urljoin(endpoint, "/profile/personal_access_tokens")
-
-login = "root"
-password = "5iveL!fe"
-
-
-def find_csrf_token(text):
-    soup = BeautifulSoup(text, "lxml")
-    token = soup.find(attrs={"name": "csrf-token"})
-    param = soup.find(attrs={"name": "csrf-param"})
-    data = {param.get("content"): token.get("content")}
-    return data
-
-
-def obtain_csrf_token():
-    r = requests.get(root_route)
-    token = find_csrf_token(r.text)
-    return token, r.cookies
-
-
-def sign_in(csrf, cookies):
-    data = {"user[login]": login, "user[password]": password}
-    data.update(csrf)
-    r = requests.post(sign_in_route, data=data, cookies=cookies)
-    token = find_csrf_token(r.text)
-    return token, r.history[0].cookies
-
-
-def obtain_personal_access_token(name, csrf, cookies):
-    data = {
-        "personal_access_token[name]": name,
-        "personal_access_token[scopes][]": ["api", "sudo"],
-    }
-    data.update(csrf)
-    r = requests.post(pat_route, data=data, cookies=cookies)
-    soup = BeautifulSoup(r.text, "lxml")
-    token = soup.find("input", id="created-personal-access-token").get("value")
-    return token
+from six.moves.urllib.parse import urljoin
+from requests_html import HTMLSession
+
+ENDPOINT = "http://localhost:8080"
+LOGIN = "root"
+PASSWORD = "5iveL!fe"
+
+
+class GitlabSession(HTMLSession):
+    def __init__(self, endpoint, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.endpoint = endpoint
+        self.csrf = None
+
+    def find_csrf_token(self, html):
+        param = html.find("meta[name=csrf-param]")[0].attrs["content"]
+        token = html.find("meta[name=csrf-token]")[0].attrs["content"]
+        self.csrf = {param: token}
+
+    def obtain_csrf_token(self):
+        r = self.get(urljoin(self.endpoint, "/"))
+        self.find_csrf_token(r.html)
+
+    def sign_in(self, login, password):
+        data = {"user[login]": login, "user[password]": password, **self.csrf}
+        r = self.post(urljoin(self.endpoint, "/users/sign_in"), data=data)
+        self.find_csrf_token(r.html)
+
+    def obtain_personal_access_token(self, name):
+        data = {
+            "personal_access_token[name]": name,
+            "personal_access_token[scopes][]": ["api", "sudo"],
+            **self.csrf,
+        }
+        r = self.post(
+            urljoin(self.endpoint, "/profile/personal_access_tokens"), data=data
+        )
+        return r.html.find("#created-personal-access-token")[0].attrs["value"]
 
 
 def main():
-    csrf1, cookies1 = obtain_csrf_token()
-    csrf2, cookies2 = sign_in(csrf1, cookies1)
-
-    token = obtain_personal_access_token("default", csrf2, cookies2)
-    print(token)
+    with GitlabSession(ENDPOINT) as s:
+        s.obtain_csrf_token()
+        s.sign_in(LOGIN, PASSWORD)
+        print(s.obtain_personal_access_token("default"))
 
 
 if __name__ == "__main__":
diff --git a/tools/python_test_v4.py b/tools/python_test_v4.py
@@ -190,7 +190,9 @@
 
 new_user.delete()
 foobar_user.delete()
-assert len(gl.users.list()) == 3
+assert len(gl.users.list()) == 3 + len(
+    [u for u in gl.users.list() if u.username == "ghost"]
+)
 
 # current user mail
 mail = gl.user.emails.create({"email": "current@user.com"})
@@ -787,9 +789,10 @@
 msg = gl.broadcastmessages.create({"message": "this is the message"})
 msg.color = "#444444"
 msg.save()
+msg_id = msg.id
 msg = gl.broadcastmessages.list(all=True)[0]
 assert msg.color == "#444444"
-msg = gl.broadcastmessages.get(1)
+msg = gl.broadcastmessages.get(msg_id)
 assert msg.color == "#444444"
 msg.delete()
 assert len(gl.broadcastmessages.list()) == 0
diff --git a/tools/reset_gitlab.py b/tools/reset_gitlab.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import sys
+
+from gitlab import Gitlab
+
+
+def main():
+    with Gitlab.from_config(config_files=["/tmp/python-gitlab.cfg"]) as gl:
+        for project in gl.projects.list():
+            project.delete()
+        for group in gl.groups.list():
+            group.delete()
+        for user in gl.users.list():
+            if user.username != "root":
+                user.delete()
+
+
+if __name__ == "__main__":
+    sys.exit(main())
