Skip to content

Commit d3c09b9

Browse files
michaelpqmichaelpq
committed
Add PGXS options to control TAP and isolation tests, take two
The following options are added for extensions: - TAP_TESTS, to allow an extention to run TAP tests which are the ones present in t/*.pl. A subset of tests can always be run with the existing PROVE_TESTS for developers. - ISOLATION, to define a list of isolation tests. - ISOLATION_OPTS, to pass custom options to isolation_tester. A couple of custom Makefile rules have been accumulated across the tree to cover the lack of facility in PGXS for a couple of releases when using those test suites, which are all now replaced with the new flags, without reducing the test coverage. Note that tests of contrib/bloom/ are not enabled yet, as those are proving unstable in the buildfarm. Author: Michael Paquier Reviewed-by: Adam Berlin, Álvaro Herrera, Tom Lane, Nikolay Shaplov, Arthur Zakirov Discussion: https://postgr.es/m/20180906014849.GG2726@paquier.xyz
1 parent 29180e5 commit d3c09b9

File tree

11 files changed

+127
-133
lines changed

11 files changed

+127
-133
lines changed

‎contrib/bloom/Makefile

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ PGFILEDESC = "bloom access method - signature file based index"
99

1010
REGRESS = bloom
1111

12+
# Disable TAP tests for this module for now, as these are unstable on several
13+
# buildfarm environments.
14+
# TAP_TESTS = 1
15+
1216
ifdef USE_PGXS
1317
PG_CONFIG = pg_config
1418
PGXS := $(shell $(PG_CONFIG) --pgxs)
@@ -19,6 +23,3 @@ top_builddir = ../..
1923
include $(top_builddir)/src/Makefile.global
2024
include $(top_srcdir)/contrib/contrib-global.mk
2125
endif
22-
23-
wal-check: temp-install
24-
$(prove_check)

‎contrib/oid2name/Makefile

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ PGAPPICON = win32
66
PROGRAM = oid2name
77
OBJS = oid2name.o $(WIN32RES)
88

9+
TAP_TESTS = 1
10+
911
PG_CPPFLAGS = -I$(libpq_srcdir)
1012
PG_LIBS_INTERNAL = $(libpq_pgport)
1113

12-
EXTRA_CLEAN = tmp_check
13-
1414
ifdef USE_PGXS
1515
PG_CONFIG = pg_config
1616
PGXS := $(shell $(PG_CONFIG) --pgxs)
@@ -21,9 +21,3 @@ top_builddir = ../..
2121
include $(top_builddir)/src/Makefile.global
2222
include $(top_srcdir)/contrib/contrib-global.mk
2323
endif
24-
25-
check:
26-
$(prove_check)
27-
28-
installcheck:
29-
$(prove_installcheck)

‎contrib/test_decoding/Makefile

Lines changed: 15 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,18 @@
33
MODULES = test_decoding
44
PGFILEDESC = "test_decoding - example of a logical decoding output plugin"
55

6-
# Note: because we don't tell the Makefile there are any regression tests,
7-
# we have to clean those result files explicitly
8-
EXTRA_CLEAN = $(pg_regress_clean_files)
6+
REGRESS = ddl xact rewrite toast permissions decoding_in_xact \
7+
decoding_into_rel binary prepared replorigin time messages \
8+
spill slot truncate
9+
ISOLATION = mxact delayed_startup ondisk_startup concurrent_ddl_dml \
10+
oldest_xmin snapshot_transfer
11+
12+
REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf
13+
ISOLATION_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf
14+
15+
# Disabled because these tests require "wal_level=logical", which
16+
# typical installcheck users do not have (e.g. buildfarm clients).
17+
NO_INSTALLCHECK = 1
918

1019
ifdef USE_PGXS
1120
PG_CONFIG = pg_config
@@ -18,52 +27,8 @@ include $(top_builddir)/src/Makefile.global
1827
include $(top_srcdir)/contrib/contrib-global.mk
1928
endif
2029

21-
# Disabled because these tests require "wal_level=logical", which
22-
# typical installcheck users do not have (e.g. buildfarm clients).
23-
installcheck:;
24-
2530
# But it can nonetheless be very helpful to run tests on preexisting
2631
# installation, allow to do so, but only if requested explicitly.
27-
installcheck-force: regresscheck-install-force isolationcheck-install-force
28-
29-
check: regresscheck isolationcheck
30-
31-
submake-regress:
32-
$(MAKE) -C $(top_builddir)/src/test/regress all
33-
34-
submake-isolation:
35-
$(MAKE) -C $(top_builddir)/src/test/isolation all
36-
37-
submake-test_decoding:
38-
$(MAKE) -C $(top_builddir)/contrib/test_decoding
39-
40-
REGRESSCHECKS=ddl xact rewrite toast permissions decoding_in_xact \
41-
decoding_into_rel binary prepared replorigin time messages \
42-
spill slot truncate
43-
44-
regresscheck: | submake-regress submake-test_decoding temp-install
45-
$(pg_regress_check) \
46-
--temp-config $(top_srcdir)/contrib/test_decoding/logical.conf \
47-
$(REGRESSCHECKS)
48-
49-
regresscheck-install-force: | submake-regress submake-test_decoding temp-install
50-
$(pg_regress_installcheck) \
51-
$(REGRESSCHECKS)
52-
53-
ISOLATIONCHECKS=mxact delayed_startup ondisk_startup concurrent_ddl_dml \
54-
oldest_xmin snapshot_transfer
55-
56-
isolationcheck: | submake-isolation submake-test_decoding temp-install
57-
$(pg_isolation_regress_check) \
58-
--temp-config $(top_srcdir)/contrib/test_decoding/logical.conf \
59-
$(ISOLATIONCHECKS)
60-
61-
isolationcheck-install-force: all | submake-isolation submake-test_decoding temp-install
62-
$(pg_isolation_regress_installcheck) \
63-
$(ISOLATIONCHECKS)
64-
65-
.PHONY: submake-test_decoding submake-regress check \
66-
regresscheck regresscheck-install-force \
67-
isolationcheck isolationcheck-install-force
68-
69-
temp-install: EXTRA_INSTALL=contrib/test_decoding
32+
installcheck-force:
33+
$(pg_regress_installcheck) $(REGRESS)
34+
$(pg_isolation_regress_installcheck) $(ISOLATION)

‎contrib/vacuumlo/Makefile

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ PGAPPICON = win32
66
PROGRAM = vacuumlo
77
OBJS = vacuumlo.o $(WIN32RES)
88

9+
TAP_TESTS = 1
10+
911
PG_CPPFLAGS = -I$(libpq_srcdir)
1012
PG_LIBS_INTERNAL = $(libpq_pgport)
1113

12-
EXTRA_CLEAN = tmp_check
13-
1414
ifdef USE_PGXS
1515
PG_CONFIG = pg_config
1616
PGXS := $(shell $(PG_CONFIG) --pgxs)
@@ -21,9 +21,3 @@ top_builddir = ../..
2121
include $(top_builddir)/src/Makefile.global
2222
include $(top_srcdir)/contrib/contrib-global.mk
2323
endif
24-
25-
check:
26-
$(prove_check)
27-
28-
installcheck:
29-
$(prove_installcheck)

‎doc/src/sgml/extend.sgml

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1303,6 +1303,34 @@ include $(PGXS)
13031303
</listitem>
13041304
</varlistentry>
13051305

1306+
<varlistentry>
1307+
<term><varname>ISOLATION</varname></term>
1308+
<listitem>
1309+
<para>
1310+
list of isolation test cases, see below for more details
1311+
</para>
1312+
</listitem>
1313+
</varlistentry>
1314+
1315+
<varlistentry>
1316+
<term><varname>ISOLATION_OPTS</varname></term>
1317+
<listitem>
1318+
<para>
1319+
additional switches to pass to
1320+
<application>pg_isolation_regress</application>
1321+
</para>
1322+
</listitem>
1323+
</varlistentry>
1324+
1325+
<varlistentry>
1326+
<term><varname>TAP_TESTS</varname></term>
1327+
<listitem>
1328+
<para>
1329+
switch defining if TAP tests need to be run, see below
1330+
</para>
1331+
</listitem>
1332+
</varlistentry>
1333+
13061334
<varlistentry>
13071335
<term><varname>NO_INSTALLCHECK</varname></term>
13081336
<listitem>
@@ -1423,13 +1451,42 @@ make VPATH=/path/to/extension/source/tree install
14231451
have all expected files.
14241452
</para>
14251453

1454+
<para>
1455+
The scripts listed in the <varname>ISOLATION</varname> variable are used
1456+
for tests stressing behavior of concurrent session with your module, which
1457+
can be invoked by <literal>make installcheck</literal> after doing
1458+
<literal>make install</literal>. For this to work you must have a
1459+
running <productname>PostgreSQL</productname> server. The script files
1460+
listed in <varname>ISOLATION</varname> must appear in a subdirectory
1461+
named <literal>specs/</literal> in your extension's directory. These files
1462+
must have extension <literal>.spec</literal>, which must not be included
1463+
in the <varname>ISOLATION</varname> list in the makefile. For each test
1464+
there should also be a file containing the expected output in a
1465+
subdirectory named <literal>expected/</literal>, with the same stem and
1466+
extension <literal>.out</literal>. <literal>make installcheck</literal>
1467+
executes each test script, and compares the resulting output to the
1468+
matching expected file. Any differences will be written to the file
1469+
<literal>output_iso/regression.diffs</literal> in
1470+
<command>diff -c</command> format. Note that trying to run a test that is
1471+
missing its expected file will be reported as <quote>trouble</quote>, so
1472+
make sure you have all expected files.
1473+
</para>
1474+
1475+
<para>
1476+
<literal>TAP_TESTS</literal> enables the use of TAP tests. Data from each
1477+
run is present in a subdirectory named <literal>tmp_check/</literal>.
1478+
See also <xref linkend="regress-tap"/> for more details.
1479+
</para>
1480+
14261481
<tip>
14271482
<para>
14281483
The easiest way to create the expected files is to create empty files,
14291484
then do a test run (which will of course report differences). Inspect
14301485
the actual result files found in the <literal>results/</literal>
1431-
directory, then copy them to <literal>expected/</literal> if they match
1432-
what you expect from the test.
1486+
directory (for tests in <literal>REGRESS</literal>), or
1487+
<literal>output_iso/results/</literal> directory (for tests in
1488+
<literal>ISOLATION</literal>), then copy them to
1489+
<literal>expected/</literal> if they match what you expect from the test.
14331490
</para>
14341491

14351492
</tip>

‎src/makefiles/pgxs.mk

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@
4646
# HEADERS_built_$(MODULE) -- as above but built first (also NOT cleaned)
4747
# REGRESS -- list of regression test cases (without suffix)
4848
# REGRESS_OPTS -- additional switches to pass to pg_regress
49+
# TAP_TESTS -- switch to enable TAP tests
50+
# ISOLATION -- list of isolation test cases
51+
# ISOLATION_OPTS -- additional switches to pass to pg_isolation_regress
4952
# NO_INSTALLCHECK -- don't define an installcheck target, useful e.g. if
5053
# tests require special configuration, or don't use pg_regress
5154
# EXTRA_CLEAN -- extra files to remove in 'make clean'
@@ -349,6 +352,12 @@ ifeq ($(PORTNAME), win)
349352
rm -f regress.def
350353
endif
351354
endif # REGRESS
355+
ifdef TAP_TESTS
356+
rm -rf tmp_check/
357+
endif
358+
ifdef ISOLATION
359+
rm -rf output_iso/ tmp_check_iso/
360+
endif
352361

353362
ifdef MODULE_big
354363
clean: clean-lib
@@ -383,28 +392,47 @@ $(test_files_build): $(abs_builddir)/%: $(srcdir)/%
383392
$(MKDIR_P) $(dir $@)
384393
ln -s $< $@
385394
endif # VPATH
395+
endif # REGRESS
386396

387397
.PHONY: submake
388398
submake:
389399
ifndef PGXS
390400
$(MAKE) -C $(top_builddir)/src/test/regress pg_regress$(X)
401+
$(MAKE) -C $(top_builddir)/src/test/isolation all
391402
endif
392403

393-
# against installed postmaster
404+
# Standard rules to run regression tests including multiple test suites.
405+
# Runs against an installed postmaster.
394406
ifndef NO_INSTALLCHECK
395407
installcheck: submake $(REGRESS_PREP)
408+
ifdef REGRESS
396409
$(pg_regress_installcheck) $(REGRESS_OPTS) $(REGRESS)
397410
endif
411+
ifdef ISOLATION
412+
$(pg_isolation_regress_installcheck) $(ISOLATION_OPTS) $(ISOLATION)
413+
endif
414+
ifdef TAP_TESTS
415+
$(prove_installcheck)
416+
endif
417+
endif # NO_INSTALLCHECK
398418

419+
# Runs independently of any installation
399420
ifdef PGXS
400421
check:
401422
@echo '"$(MAKE) check" is not supported.'
402423
@echo 'Do "$(MAKE) install", then "$(MAKE) installcheck" instead.'
403424
else
404425
check: submake $(REGRESS_PREP)
426+
ifdef REGRESS
405427
$(pg_regress_check) $(REGRESS_OPTS) $(REGRESS)
406428
endif
407-
endif # REGRESS
429+
ifdef ISOLATION
430+
$(pg_isolation_regress_check) $(ISOLATION_OPTS) $(ISOLATION)
431+
endif
432+
ifdef TAP_TESTS
433+
$(prove_check)
434+
endif
435+
endif # PGXS
408436

409437
ifndef NO_TEMP_INSTALL
410438
temp-install: EXTRA_INSTALL+=$(subdir)

‎src/test/modules/brin/.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# Generated subdirectories
2-
/isolation_output/
2+
/output_iso/
33
/tmp_check/

‎src/test/modules/brin/Makefile

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
# src/test/modules/brin/Makefile
22

3-
# Note: because we don't tell the Makefile there are any regression tests,
4-
# we have to clean those result files explicitly
5-
EXTRA_CLEAN = $(pg_regress_clean_files) ./isolation_output
3+
EXTRA_INSTALL = contrib/pageinspect
64

7-
EXTRA_INSTALL=contrib/pageinspect
8-
9-
ISOLATIONCHECKS=summarization-and-inprogress-insertion
5+
ISOLATION = summarization-and-inprogress-insertion
6+
TAP_TESTS = 1
107

118
ifdef USE_PGXS
129
PG_CONFIG = pg_config
@@ -18,19 +15,3 @@ top_builddir = ../../../..
1815
include $(top_builddir)/src/Makefile.global
1916
include $(top_srcdir)/contrib/contrib-global.mk
2017
endif
21-
22-
check: isolation-check prove-check
23-
24-
isolation-check: | submake-isolation temp-install
25-
$(MKDIR_P) isolation_output
26-
$(pg_isolation_regress_check) \
27-
--outputdir=./isolation_output \
28-
$(ISOLATIONCHECKS)
29-
30-
prove-check: | temp-install
31-
$(prove_check)
32-
33-
.PHONY: check isolation-check prove-check
34-
35-
submake-isolation:
36-
$(MAKE) -C $(top_builddir)/src/test/isolation all

‎src/test/modules/commit_ts/Makefile

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/commit_ts/commit_ts.
66
# which typical installcheck users do not have (e.g. buildfarm clients).
77
NO_INSTALLCHECK = 1
88

9+
TAP_TESTS = 1
10+
911
ifdef USE_PGXS
1012
PG_CONFIG = pg_config
1113
PGXS := $(shell $(PG_CONFIG) --pgxs)
@@ -16,8 +18,3 @@ top_builddir = ../../../..
1618
include $(top_builddir)/src/Makefile.global
1719
include $(top_srcdir)/contrib/contrib-global.mk
1820
endif
19-
20-
check: prove-check
21-
22-
prove-check: | temp-install
23-
$(prove_check)

0 commit comments

Comments
 (0)