Fix ancient violation of zlib's API spec.

tglsfdc · tglsfdc · commit 3d4a778152ec · 2020-07-23T17:20:02.000-04:00
contrib/pgcrypto mishandled the case where deflate() does not consume all of the offered input on the first try. It reset the next_in pointer to the start of the input instead of leaving it alone, causing the wrong data to be fed to the next deflate() call. This has been broken since pgcrypto was committed. The reason for the lack of complaints seems to be that it's fairly hard to get stock zlib to not consume all the input, so long as the output buffer is big enough (which it normally would be in pgcrypto's usage; AFAICT the input is always going to be packetized into packets no larger than ZIP_OUT_BUF). However, IBM's zlibNX implementation for AIX evidently will do it in some cases. I did not add a test case for this, because I couldn't find one that would fail with stock zlib. When we put back the test case for bug #16476, that will cover the zlibNX situation well enough. While here, write deflate()'s second argument as Z_NO_FLUSH per its API spec, instead of hard-wiring the value zero. Per buildfarm results and subsequent investigation. Discussion: https://postgr.es/m/16476-692ef7b84e5fb893@postgresql.org
diff --git a/contrib/pgcrypto/pgp-compress.c b/contrib/pgcrypto/pgp-compress.c
@@ -115,13 +115,13 @@ compress_process(PushFilter *next, void *priv, const uint8 *data, int len)
 	/*
 	 * process data
 	 */
-	while (len > 0)
+	st->stream.next_in = unconstify(uint8 *, data);
+	st->stream.avail_in = len;
+	while (st->stream.avail_in > 0)
 	{
-		st->stream.next_in = unconstify(uint8 *, data);
-		st->stream.avail_in = len;
 		st->stream.next_out = st->buf;
 		st->stream.avail_out = st->buf_len;
-		res = deflate(&st->stream, 0);
+		res = deflate(&st->stream, Z_NO_FLUSH);
 		if (res != Z_OK)
 			return PXE_PGP_COMPRESSION_ERROR;
 
@@ -132,7 +132,6 @@ compress_process(PushFilter *next, void *priv, const uint8 *data, int len)
 			if (res < 0)
 				return res;
 		}
-		len = st->stream.avail_in;
 	}
 
 	return 0;
@@ -155,6 +154,7 @@ compress_flush(PushFilter *next, void *priv)
 		zres = deflate(&st->stream, Z_FINISH);
 		if (zres != Z_STREAM_END && zres != Z_OK)
 			return PXE_PGP_COMPRESSION_ERROR;
+
 		n_out = st->buf_len - st->stream.avail_out;
 		if (n_out > 0)
 		{
