From 3540fc907a461f53d7a95b2732d1749ab333f7a6 Mon Sep 17 00:00:00 2001 From: TyrannosourceExe Date: Wed, 16 May 2018 10:15:10 -0400 Subject: [PATCH 1/4] patched string index out of range error in get_word function of _header_value_parser.py and created tests in test__header_value_parser.py for CFWS. --- Lib/email/_header_value_parser.py | 16 ++++++++---- .../test_email/test__header_value_parser.py | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index bb5ff8dc7e8034..524ceb982196d5 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1360,15 +1360,21 @@ def get_word(value): leader, value = get_cfws(value) else: leader = None - if value[0]=='"': - token, value = get_quoted_string(value) - elif value[0] in SPECIALS: - raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " - "but found '{}'".format(value)) + if value: + if value[0]=='"': + token, value = get_quoted_string(value) + elif value[0] in SPECIALS: + raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " + "but found '{}'".format(value)) + else: + token, value = get_atom(value) + if leader is not None: + token[:0] = [leader] else: token, value = get_atom(value) if leader is not None: token[:0] = [leader] + return token, value def get_phrase(value): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 649923fa6c8667..8712cd11bb18d0 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -930,6 +930,20 @@ def test_get_word_atom_yields_atom(self): self.assertEqual(word.token_type, 'atom') self.assertEqual(word[0].token_type, 'cfws') + def test_get_word_all_CFWS(self): + word = self._test_get_x(parser.get_word, + '(Recipients list suppressed', + str(parser.CFWSList([parser.Comment([ + parser.WhiteSpaceTerminal('Recipients', 'ptext'), + parser.WhiteSpaceTerminal(' ', 'fws'), + parser.WhiteSpaceTerminal('list', 'ptext'), + parser.WhiteSpaceTerminal(' ', 'fws'), + parser.WhiteSpaceTerminal('suppressed', 'ptext') + ])])), + ' ', [], '' + ) + self.assertEqual(word.token_type, 'cfws') + def test_get_word_qs_yields_qs(self): word = self._test_get_x(parser.get_word, '"bar " (bang) ah', '"bar " (bang) ', 'bar ', [], 'ah') @@ -2343,6 +2357,17 @@ def test_get_address_quoted_strings_in_atom_list(self): # get_address_list + def test_get_address_list_CFWS(self): + address_list = self._test_get_x(parser.get_address_list, + '(Recipient list suppressed)', + '(Recipient list suppressed)', + ' ', + [errors.ObsoleteHeaderDefect], # no content in address list + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 0) + self.assertEqual(address_list.mailboxes, address_list.all_mailboxes) + def test_get_address_list_mailboxes_simple(self): address_list = self._test_get_x(parser.get_address_list, 'dinsdale@example.com', From 2078cc8b9beb8fcca63d2a2c341f91aee39da787 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Tue, 25 Jun 2019 19:27:33 -0700 Subject: [PATCH 2/4] Raise HeaderParseError instead of continuing when parsing a word. Also, modify the testcase. --- Lib/email/_header_value_parser.py | 17 +++++++---------- .../test_email/test__header_value_parser.py | 16 ++++------------ .../2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst | 2 ++ 3 files changed, 13 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 524ceb982196d5..88cf08c1dbf1b9 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1360,16 +1360,13 @@ def get_word(value): leader, value = get_cfws(value) else: leader = None - if value: - if value[0]=='"': - token, value = get_quoted_string(value) - elif value[0] in SPECIALS: - raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " - "but found '{}'".format(value)) - else: - token, value = get_atom(value) - if leader is not None: - token[:0] = [leader] + if not value: + raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " + "but found nothing.") + if value[0]=='"': + token, value = get_quoted_string(value) + elif value[0] in SPECIALS: + raise errors.HeaderParseError((value)) else: token, value = get_atom(value) if leader is not None: diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 8712cd11bb18d0..c4e1a9f99495a4 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -931,18 +931,10 @@ def test_get_word_atom_yields_atom(self): self.assertEqual(word[0].token_type, 'cfws') def test_get_word_all_CFWS(self): - word = self._test_get_x(parser.get_word, - '(Recipients list suppressed', - str(parser.CFWSList([parser.Comment([ - parser.WhiteSpaceTerminal('Recipients', 'ptext'), - parser.WhiteSpaceTerminal(' ', 'fws'), - parser.WhiteSpaceTerminal('list', 'ptext'), - parser.WhiteSpaceTerminal(' ', 'fws'), - parser.WhiteSpaceTerminal('suppressed', 'ptext') - ])])), - ' ', [], '' - ) - self.assertEqual(word.token_type, 'cfws') + # bpo-29412: Test that we don't raise IndexError when parsing CFWS only + # token. + with self.assertRaises(errors.HeaderParseError): + parser.get_word('(Recipients list suppressed') def test_get_word_qs_yields_qs(self): word = self._test_get_x(parser.get_word, diff --git a/Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst b/Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst new file mode 100644 index 00000000000000..717242b18d2fe9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst @@ -0,0 +1,2 @@ +Fix IndexError in parsing a header value with only comments in some headers. +Patch by Abhilash Raj. From 661962a464499978b331695fbe21f2ed2028484e Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Tue, 25 Jun 2019 22:27:33 -0700 Subject: [PATCH 3/4] Undo code changed by mistake. --- Lib/email/_header_value_parser.py | 4 ++-- .../next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 88cf08c1dbf1b9..2460e3e2d483c6 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1366,12 +1366,12 @@ def get_word(value): if value[0]=='"': token, value = get_quoted_string(value) elif value[0] in SPECIALS: - raise errors.HeaderParseError((value)) + raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " + "but found '{}'".format(value)) else: token, value = get_atom(value) if leader is not None: token[:0] = [leader] - return token, value def get_phrase(value): diff --git a/Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst b/Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst index 717242b18d2fe9..b8fac46736867b 100644 --- a/Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst +++ b/Misc/NEWS.d/next/Library/2019-06-25-19-27-25.bpo-29412.n4Zqdh.rst @@ -1,2 +1,2 @@ -Fix IndexError in parsing a header value with only comments in some headers. -Patch by Abhilash Raj. +Fix IndexError in parsing a header value ending unexpectedly. Patch by Abhilash +Raj. From a3cc163b54b4e382913ac11f6ab43a489505689e Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Wed, 26 Jun 2019 11:03:25 -0700 Subject: [PATCH 4/4] Remove implicit concatenation Remove implicit concatenation of strings and move the string to next line so that it can fit properly under 79 chars. --- Lib/email/_header_value_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 2460e3e2d483c6..aefc457f678165 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1361,8 +1361,8 @@ def get_word(value): else: leader = None if not value: - raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " - "but found nothing.") + raise errors.HeaderParseError( + "Expected 'atom' or 'quoted-string' but found nothing.") if value[0]=='"': token, value = get_quoted_string(value) elif value[0] in SPECIALS: