pg_regress: Emit TAP compliant output

danielgustafsson · danielgustafsson · commit 558fff0adfa0 · 2023-03-31T13:00:02.000+02:00
This converts pg_regress output format to emit TAP compliant output while keeping it as human readable as possible for use without TAP test harnesses. As verbose harness related information isn't really supported by TAP this also reduces the verbosity of pg_regress runs which makes scrolling through log output in buildfarm/CI runs a bit easier as well. As the meson TAP parser conumes whitespace, the leading indentation for differentiating parallel tests from sequential tests has been changed to a single character prefix. This patch has been around for an extended period of time, reviewers listed below may have been involved in reviewing a version quite different from the version in this commit. The original idea for this patch was a hacking session with Jinbao Chen. TAP format testing is also enabled in meson as of this. Reviewed-by: Andres Freund <andres@anarazel.de> Reviewed-by: Tom Lane <tgl@sss.pgh.pa.us> Reviewed-by: Nikolay Shaplov <dhyan@nataraj.su> Reviewed-by: Dagfinn Ilmari Mannsåker <ilmari@ilmari.org> Reviewed-by: Peter Eisentraut <peter.eisentraut@enterprisedb.com> Discussion: https://postgr.es/m/BD4B107D-7E53-4794-ACBA-275BEB4327C9@yesql.se Discussion: https://postgr.es/m/20220221164736.rq3ornzjdkmwk2wo@alap3.anarazel.de
diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml
@@ -30,6 +30,8 @@
    parallel method starts up multiple server processes to run groups
    of tests in parallel.  Parallel testing adds confidence that
    interprocess communication and locking are working correctly.
+   Some tests may run sequentially even in the <quote>parallel</quote>
+   mode in case this is required by the test.
   </para>
 
   <sect2 id="regress-run-temp-inst">
@@ -43,12 +45,12 @@ make check
 </screen>
    in the top-level directory.  (Or you can change to
    <filename>src/test/regress</filename> and run the command there.)
+   Tests which are run in parallel are prefixed with <quote>+</quote>, and
+   tests which run sequentially are prefixed with <quote>-</quote>.
    At the end you should see something like:
 <screen>
 <computeroutput>
-=======================
- All 193 tests passed.
-=======================
+# All 213 tests passed.
 </computeroutput>
 </screen>
    or otherwise a note about which tests failed.  See <xref
diff --git a/meson.build b/meson.build
@@ -3118,6 +3118,7 @@ foreach test_dir : tests
       env.prepend('PATH', temp_install_bindir, test_dir['bd'])
 
       test_kwargs = {
+        'protocol': 'tap',
         'priority': 10,
         'timeout': 1000,
         'depends': test_deps + t.get('deps', []),
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
@@ -444,7 +444,7 @@ ifeq ($(enable_tap_tests),yes)
 
 ifndef PGXS
 define prove_installcheck
-echo "+++ tap install-check in $(subdir) +++" && \
+echo "# +++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
@@ -457,7 +457,7 @@ cd $(srcdir) && \
 endef
 else # PGXS case
 define prove_installcheck
-echo "+++ tap install-check in $(subdir) +++" && \
+echo "# +++ tap install-check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
@@ -471,7 +471,7 @@ endef
 endif # PGXS
 
 define prove_check
-echo "+++ tap check in $(subdir) +++" && \
+echo "# +++ tap check in $(subdir) +++" && \
 rm -rf '$(CURDIR)'/tmp_check && \
 $(MKDIR_P) '$(CURDIR)'/tmp_check && \
 cd $(srcdir) && \
@@ -665,7 +665,7 @@ pg_regress_locale_flags = $(if $(ENCODING),--encoding=$(ENCODING)) $(NOLOCALE)
 pg_regress_clean_files = results/ regression.diffs regression.out tmp_check/ tmp_check_iso/ log/ output_iso/
 
 pg_regress_check = \
-    echo "+++ regress check in $(subdir) +++" && \
+    echo "\# +++ regress check in $(subdir) +++" && \
     $(with_temp_install) \
     $(top_builddir)/src/test/regress/pg_regress \
     --temp-instance=./tmp_check \
@@ -674,14 +674,14 @@ pg_regress_check = \
     $(TEMP_CONF) \
     $(pg_regress_locale_flags) $(EXTRA_REGRESS_OPTS)
 pg_regress_installcheck = \
-    echo "+++ regress install-check in $(subdir) +++" && \
+    echo "\# +++ regress install-check in $(subdir) +++" && \
     $(top_builddir)/src/test/regress/pg_regress \
     --inputdir=$(srcdir) \
     --bindir='$(bindir)' \
     $(pg_regress_locale_flags) $(EXTRA_REGRESS_OPTS)
 
 pg_isolation_regress_check = \
-    echo "+++ isolation check in $(subdir) +++" && \
+    echo "\# +++ isolation check in $(subdir) +++" && \
     $(with_temp_install) \
     $(top_builddir)/src/test/isolation/pg_isolation_regress \
     --temp-instance=./tmp_check_iso \
@@ -690,7 +690,7 @@ pg_isolation_regress_check = \
     $(TEMP_CONF) \
     $(pg_regress_locale_flags) $(EXTRA_REGRESS_OPTS)
 pg_isolation_regress_installcheck = \
-    echo "+++ isolation install-check in $(subdir) +++" && \
+    echo "\# +++ isolation install-check in $(subdir) +++" && \
     $(top_builddir)/src/test/isolation/pg_isolation_regress \
     --inputdir=$(srcdir) --outputdir=output_iso \
     --bindir='$(bindir)' \
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
