Skip to content

Commit 2d10346

Browse files
vbwagnervbwagner
committed
gitlab CI: split into many stages, added helper scripts for collecting logs and setup temporary installation for make installcheck
1 parent 2fb43b9 commit 2d10346

File tree

5 files changed

+227
-15
lines changed

5 files changed

+227
-15
lines changed

‎.ci/collect_logs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env python
2+
"""
3+
Prints out all the files with .log extension
4+
"""
5+
import os,re,subprocess
6+
cores=[]
7+
for root,dirs,files in os.walk("."):
8+
for name in files:
9+
path= root+"/"+name
10+
if name.endswith(".log") or name=="regression.diffs":
11+
print "*" * 76
12+
print "* %-72s *"%path
13+
print "*" * 76
14+
with open(path,"r") as f:
15+
print f.read()
16+
if name == "core" or re.match("core\\.[0-9]+",name) or re.match(".*\\.core",name):
17+
cores.append(path)
18+
print ", ".join(cores)
19+
for f in cores:
20+
print "*" * 76
21+
print "* %-72s *"%f
22+
print "*" * 76
23+
gdb=subprocess.Popen(["gdb","src/backend/postgres",f],
24+
stderr=subprocess.PIPE,stdout=subprocess.PIPE,stdin=subprocess.PIPE)
25+
out,err=gdb.communicate("bt\nq\n")
26+
if err.index("core file may not match specified")>=0:
27+
m=re.search("Core was generated by `(.*)'",out)
28+
if m:
29+
gdb=subprocess.Popen(["gdb",m.group(1),f],
30+
stdout=subprocess.PIPE,stdin=subprocess.PIPE)
31+
out,err=gdb.communicate("bt\nq\n")
32+
33+
34+
print out
35+
36+
37+
38+

‎.ci/find_free_port

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/python
2+
import sys,socket,errno
3+
if len(sys.argv)>1:
4+
port=int(sys.argv[1])
5+
else:
6+
port=5900
7+
8+
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
9+
while True:
10+
try:
11+
s.bind(("",port))
12+
except socket.error as e:
13+
if e.errno== errno.EADDRINUSE:
14+
port+=1
15+
continue
16+
else:
17+
raise e
18+
break
19+
20+
s.close()
21+
print port
22+

‎.ci/make_test_base

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env python
2+
"""
3+
Receives one parameter - PGDATA directory
4+
Creates test base using first initdb found in the PATH
5+
Searches contrib directory for additional directives to
6+
put into postgresql conf.
7+
8+
On success, starts database
9+
"""
10+
import os,sys,subprocess,glob,re,os.path,time
11+
12+
datadir=sys.argv[1]
13+
14+
if not datadir:
15+
print >>sys.stderr,"Usage %s directory"%sys.argv[0]
16+
sys.exit(1)
17+
18+
if os.access(datadir,os.R_OK):
19+
import shutil
20+
shutil.rmtree(datadir)
21+
os.mkdir(datadir)
22+
with open("initdb.log","w") as f:
23+
exitcode=subprocess.call(["initdb","-E","UTF8",datadir],stdout=f,stderr=subprocess.STDOUT)
24+
if exitcode:
25+
sys.exit(exitcode)
26+
# Collect extra config option
27+
addopts={}
28+
for module in glob.glob("contrib/*"):
29+
if not os.path.isdir(module):
30+
continue
31+
if not os.access(module+"/Makefile",os.R_OK):
32+
continue
33+
with open(module+"/Makefile","r") as makefile:
34+
var={"top_srcdir":os.getcwd()}
35+
for line in makefile:
36+
m=re.match("\s*(\w+)\s*=\s*(.*)",line)
37+
if m:
38+
var[m.group(1)]=m.group(2)
39+
if "EXTRA_REGRESS_OPTS" in var:
40+
m=re.search("--temp-config=(\S+)",var["EXTRA_REGRESS_OPTS"])
41+
if m:
42+
filename=re.sub("\\$[{(](\w+)[})]",lambda m: var[m.group(1)],m.group(1))
43+
with open(filename,"r") as config:
44+
for line in config:
45+
m=re.match("(\w\S*\w)\s*=\s*(\S.*)\s*",line)
46+
if m:
47+
opt=m.group(1)
48+
value=m.group(2)
49+
if opt in addopts:
50+
if value[0]=="'":
51+
addopts[opt]=addopts[opt][:-1]+", "+value[1:]
52+
else:
53+
addopts[opt]+=", ".value
54+
else:
55+
addopts[opt]=value
56+
57+
if addopts:
58+
with open(datadir+"/postgresql.conf","a") as f:
59+
for opt,value in addopts.items():
60+
print >> f,"%s=%s"%(opt,value)
61+
with open("initdb.log","a") as f:
62+
exitcode=subprocess.call(["pg_ctl","start","-D",datadir,"-l",datadir+"/postmaster.log"],stdout=f,stderr=subprocess.STDOUT)
63+
if exitcode:
64+
sys.exit(exitcode)
65+
66+
failtime=time.time()+60
67+
print "Waiting for database to start"
68+
while time.time() < failtime:
69+
exitcode=subprocess.call(["psql","postgres","-c","select version()"],stderr=subprocess.STDOUT)
70+
if exitcode == 0:
71+
sys.exit(0)
72+
print >>sys.stderr,"Database havent't started in 60 seconds"
73+
sys.exit(1)
74+
75+

‎.ci/run_install_check

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/sh
2+
# Makes temporary installation and runs installcheck-world on it
3+
# Should be run as non-privileged user.
4+
# Exit on all errors
5+
set -e
6+
# Install
7+
make install prefix=`pwd`/tmp_install
8+
make install prefix=`pwd`/tmp_install -C contrib
9+
10+
# Setup an environment
11+
LD_LIBRARY_PATH=$(pwd)/tmp_install/lib
12+
PATH=$(pwd)/tmp_install/bin:${PATH}
13+
PGDATA=$(pwd)/tmp_base
14+
export LD_LIBRARY_PATH PATH PG_DATA
15+
16+
# create installation
17+
PGPORT=`./.ci/find_free_port 5432`
18+
export PGPORT
19+
./.ci/make_test_base $PGDATA
20+
#run checks
21+
set +e
22+
make installcheck-world prefix=`pwd`/tmp_install
23+
code=$?
24+
pg_ctl stop -D $PGDATA
25+
exit $code

‎.gitlab-ci.yml

Lines changed: 67 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,79 @@
1-
image: debian
2-
31
stages:
2+
- prepare
3+
- build
44
- test
5+
- installtest
6+
- collect_logs
57

6-
before_script:
8+
prepare-doc:
9+
stage: prepare
10+
image: ubuntu:16.04
11+
tags:
12+
- deb
13+
- docker
14+
only:
15+
- /^PGPRO.*9_[56]$/
16+
before_script:
717
- export DEBIAN_FRONTEND=noninteractive
818
- export CORES=$(grep -c ^processor /proc/cpuinfo)
919
- uname -a
1020
- df -h
11-
12-
test:ubuntu-16.04:
13-
stage: test
14-
image: ubuntu:16.04
15-
only:
16-
- PGPROEE9_6
17-
before_script:
21+
script:
1822
- echo "Current user id is `id`"
1923
- apt-get update && apt-get install -y sudo gcc make flex bison libreadline-dev zlib1g-dev openjade libzstd-dev opensp docbook docbook-xml docbook-xsl libxml2-utils xsltproc python-dev libicu-dev
2024
- id -u postgres || adduser --disabled-login --gecos "Postgres" postgres
25+
build_job:
26+
stage: build
27+
tags:
28+
- unix
29+
only:
30+
- /^PGPRO.*9_[56]$/
2131
script:
22-
- ./configure --prefix=/opt/pgproee --with-zstd --with-icu --with-python
23-
- make -j $CORES world
24-
- sudo -u postgres make check-world
25-
after_script:
26-
- cat src/test/regress/log/initdb.log
32+
- if [ -z "$PGPORT" ]; then PGPORT=5789; fi
33+
- ./configure --prefix=$(pwd)/tmp_install --port=--with-zstd --with-icu --with-python
34+
- make
35+
- make -C contrib
36+
when: always
37+
38+
build_docs_job:
39+
stage: build
2740
when: always
41+
only:
42+
- /^PGPRO.*9_[56]$/
43+
tags:
44+
- docs
45+
script:
46+
- make -C doc
47+
48+
make_check_job:
49+
stage: test
50+
when: on_success
51+
only:
52+
- /^PGPRO.*9_[56]$/
53+
tags:
54+
- unix
55+
before_script:
56+
- unset SUCMD
57+
- if [ $(id -u) -eq 0 ]; then SUCMD="sudo -u postgres"; fi
58+
- export SUCMD
59+
script:
60+
- $SUCMD make check-world
61+
62+
make_installcheck_job:
63+
stage: installtest
64+
when: on_success
65+
only:
66+
- /^PGPRO.*9_[56]$/
67+
tags:
68+
- unix
69+
before_script:
70+
- unset SUCMD
71+
- if [ $(id -u) -eq 0 ]; then SUCMD="sudo -u postgres"; fi
72+
- export SUCMD
73+
script:
74+
- $SUCMD ./.ci/run_install_check
75+
collect_logs_job:
76+
stage: collect_logs
77+
when: on_failure
78+
script:
79+
- ./.ci/collect_logs

0 commit comments

Comments
 (0)