From 617e06403d65b743ff7f2ffe83bd8cf9bb0accb8 Mon Sep 17 00:00:00 2001
From: Ruben Vorderman <r.h.p.vorderman@lumc.nl>
Date: Mon, 18 Oct 2021 13:47:27 +0200
Subject: [PATCH 1/3] Check gzip headers for corrupted fields

---
 Lib/gzip.py                                   | 53 +++++++++++++++----
 Lib/test/test_gzip.py                         | 30 +++++++++++
 .../2021-10-18-13-46-55.bpo-45509.Upwb60.rst  |  1 +
 3 files changed, 73 insertions(+), 11 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst

diff --git a/Lib/gzip.py b/Lib/gzip.py
index 6773ea3eef0971..693ac425d8d5f3 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -426,29 +426,60 @@ def _read_gzip_header(fp):
 
     if magic != b'\037\213':
         raise BadGzipFile('Not a gzipped file (%r)' % magic)
-
-    (method, flag, last_mtime) = struct.unpack("<BBIxx", _read_exact(fp, 8))
+    base_header = _read_exact(fp, 8)
+    (method, flag, last_mtime) = struct.unpack("<BBIxx", base_header)
     if method != 8:
         raise BadGzipFile('Unknown compression method')
 
+    # FHCRC will be checked often. So save the result of the check.
+    fhcrc = flag & FHCRC
+    # Only create and append to a list of header parts when FHCRC is set.
+    # In the most common use cases FHCRC is not set. So we optimize for those
+    # cases.
+    if fhcrc:
+        header_parts = [magic, base_header]
+
     if flag & FEXTRA:
-        # Read & discard the extra field, if present
-        extra_len, = struct.unpack("<H", _read_exact(fp, 2))
-        _read_exact(fp, extra_len)
+        # Read the extra field, if present, save the fields if FHCRC is set.
+        extra_len_bytes = _read_exact(fp, 2)
+        extra_len, = struct.unpack("<H", extra_len_bytes)
+        extra = _read_exact(fp, extra_len)
+        if fhcrc:
+            header_parts.extend([extra_len_bytes, extra])
+
     if flag & FNAME:
-        # Read and discard a null-terminated string containing the filename
+        # Read a null-terminated string containing the filename. Save it
+        # if FHCRC is set.
         while True:
             s = fp.read(1)
-            if not s or s==b'\000':
+            if not s:
+                raise EOFError("Compressed file ended before the "
+                               "end-of-stream marker was reached")
+            if fhcrc:
+                header_parts.append(s)
+            if s == b'\000':
                 break
     if flag & FCOMMENT:
-        # Read and discard a null-terminated string containing a comment
+        # Read a null-terminated string containing the filename. Save it
+        # if FHCRC is set.
         while True:
             s = fp.read(1)
-            if not s or s==b'\000':
+            if not s:
+                raise EOFError("Compressed file ended before the "
+                               "end-of-stream marker was reached")
+            if fhcrc:
+                header_parts.append(s)
+            if s == b'\000':
                 break
-    if flag & FHCRC:
-        _read_exact(fp, 2)     # Read & discard the 16-bit header CRC
+
+    if fhcrc:
+        # Read the 16-bit header CRC and check it against the header.
+        header_crc, = struct.unpack("<H", _read_exact(fp, 2))
+        header = b"".join(header_parts)
+        true_crc = zlib.crc32(header) & 0xFFFF
+        if header_crc != true_crc:
+            raise BadGzipFile(f"Corrupted gzip header. Checksums do not "
+                               f"match: {true_crc:04x} != {header_crc:04x}")
     return last_mtime
 
 
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
index aa66d2f07f508c..6476921b7525a9 100644
--- a/Lib/test/test_gzip.py
+++ b/Lib/test/test_gzip.py
@@ -9,6 +9,7 @@
 import struct
 import sys
 import unittest
+import zlib
 from subprocess import PIPE, Popen
 from test.support import import_helper
 from test.support import os_helper
@@ -570,6 +571,35 @@ def test_decompress_missing_trailer(self):
         compressed_data = gzip.compress(data1)
         self.assertRaises(EOFError, gzip.decompress, compressed_data[:-8])
 
+    def test_truncated_header(self):
+        truncated_headers = [
+            b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00",             # Missing OS byte
+            b"\x1f\x8b\x08\x02\x00\x00\x00\x00\x00\xff",         # FHRC, but no checksum
+            b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff",         # FEXTRA, but no xlen
+            b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff\xaa\x00", # FEXTRA, xlen, but no data
+            b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff",         # FNAME but no fname
+            b"\x1f\x8b\x08\x10\x00\x00\x00\x00\x00\xff",         # FCOMMENT, but no fcomment
+        ]
+        for header in truncated_headers:
+            with self.subTest(header=header):
+                with self.assertRaises(EOFError):
+                    gzip.decompress(header)
+
+    def test_corrupted_gzip_header(self):
+        header = (b"\x1f\x8b\x08\x1f\x00\x00\x00\x00\x00\xff"  # All flags set
+                  b"\x05\x00"  # Xlen = 5
+                  b"extra"
+                  b"name\x00"
+                  b"comment\x00")
+        true_crc = zlib.crc32(header) & 0xFFFF
+        corrupted_crc = true_crc ^ 0xFFFF
+        corrupted_header = header + corrupted_crc.to_bytes(2, "little")
+        with self.assertRaises(gzip.BadGzipFile) as err:
+            gzip.decompress(corrupted_header)
+        self.assertEqual(str(err.exception),
+                         f"Corrupted gzip header. Checksums do not "
+                         f"match: {true_crc:04x} != {corrupted_crc:04x}")
+
     def test_read_truncated(self):
         data = data1*50
         # Drop the CRC (4 bytes) and file size (4 bytes).
diff --git a/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst b/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst
new file mode 100644
index 00000000000000..80c38c03f8fe78
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst
@@ -0,0 +1 @@
+Gzip headers are now checked for corrupted NAME, COMMENT and HCRC fields.

From 3d5bb4726fd4122233849d2f47e00f97b6aff2ea Mon Sep 17 00:00:00 2001
From: Ruben Vorderman <r.h.p.vorderman@lumc.nl>
Date: Wed, 24 Nov 2021 10:48:00 +0100
Subject: [PATCH 2/3] Minor performance tweaks to _read_gzip_header

Call the bool method and cache the result for faster truth checking.
Do not test for empty bytes but use "not magic" instead for faster truth checking.
---
 Lib/gzip.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Lib/gzip.py b/Lib/gzip.py
index 693ac425d8d5f3..132719a7c6959a 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -421,7 +421,7 @@ def _read_gzip_header(fp):
     Returns last mtime if header was present or None otherwise.
     '''
     magic = fp.read(2)
-    if magic == b'':
+    if not magic:
         return None
 
     if magic != b'\037\213':
@@ -432,7 +432,7 @@ def _read_gzip_header(fp):
         raise BadGzipFile('Unknown compression method')
 
     # FHCRC will be checked often. So save the result of the check.
-    fhcrc = flag & FHCRC
+    fhcrc = bool(flag & FHCRC)
     # Only create and append to a list of header parts when FHCRC is set.
     # In the most common use cases FHCRC is not set. So we optimize for those
     # cases.

From e68e76ede3cad99c094531ccf0596ad70bf1c883 Mon Sep 17 00:00:00 2001
From: Ruben Vorderman <r.h.p.vorderman@lumc.nl>
Date: Wed, 24 Nov 2021 11:42:40 +0100
Subject: [PATCH 3/3] Optimize _read_gzip_header for the most common code paths

Those are:
+ Only FNAME set. (Created by gzip and python's GzipFile)
+ No flags set. (Created by gzip.compress and zlib.compress with wbits=31)
---
 Lib/gzip.py | 48 +++++++++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/Lib/gzip.py b/Lib/gzip.py
index 132719a7c6959a..171b8ba77c0d1a 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -431,48 +431,58 @@ def _read_gzip_header(fp):
     if method != 8:
         raise BadGzipFile('Unknown compression method')
 
-    # FHCRC will be checked often. So save the result of the check.
-    fhcrc = bool(flag & FHCRC)
-    # Only create and append to a list of header parts when FHCRC is set.
-    # In the most common use cases FHCRC is not set. So we optimize for those
-    # cases.
-    if fhcrc:
-        header_parts = [magic, base_header]
+    # No flags. No need for further parsing. These headers are returned by
+    # gzip.compress or zlib.compress(..., wbits=31)
+    if not flag:
+        return last_mtime
+    # Most gzip files will have only FNAME set. For example: produced by gzip
+    # command line application or python's GzipFile.
+    if flag == FNAME:
+        while True:
+            s = fp.read(1)
+            if not s:
+                raise EOFError("Compressed file ended before the "
+                               "end-of-stream marker was reached")
+            if s == b'\000':
+                break
+        return last_mtime
+
+    # Processing for more complex flags.
+
+    # Save header parts for FHCRC checking
+    header_parts = [magic, base_header]
 
     if flag & FEXTRA:
-        # Read the extra field, if present, save the fields if FHCRC is set.
+        # Read the extra field, if present, save the fields for FHCRC checking.
         extra_len_bytes = _read_exact(fp, 2)
         extra_len, = struct.unpack("<H", extra_len_bytes)
         extra = _read_exact(fp, extra_len)
-        if fhcrc:
-            header_parts.extend([extra_len_bytes, extra])
+        header_parts.extend([extra_len_bytes, extra])
 
     if flag & FNAME:
-        # Read a null-terminated string containing the filename. Save it
-        # if FHCRC is set.
+        # Read a null-terminated string containing the filename, save the name
+        # for FHCRC checking.
         while True:
             s = fp.read(1)
             if not s:
                 raise EOFError("Compressed file ended before the "
                                "end-of-stream marker was reached")
-            if fhcrc:
-                header_parts.append(s)
+            header_parts.append(s)
             if s == b'\000':
                 break
     if flag & FCOMMENT:
-        # Read a null-terminated string containing the filename. Save it
-        # if FHCRC is set.
+        # Read a null-terminated string containing the comment, save the
+        # comment for FHCRC checking.
         while True:
             s = fp.read(1)
             if not s:
                 raise EOFError("Compressed file ended before the "
                                "end-of-stream marker was reached")
-            if fhcrc:
-                header_parts.append(s)
+            header_parts.append(s)
             if s == b'\000':
                 break
 
-    if fhcrc:
+    if flag & FHCRC:
         # Read the 16-bit header CRC and check it against the header.
         header_crc, = struct.unpack("<H", _read_exact(fp, 2))
         header = b"".join(header_parts)