From 86530c7a8de18d1cffb20b746d38ce9680f91d40 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev Date: Fri, 25 Sep 2026 10:48:49 +0300 Subject: [PATCH 1/2] gh-66889: Report a defect on the misplaced-special token in an invalid mailbox get_invalid_mailbox() builds a 'misplaced-special' ValueTerminal for every stray special character but leaves its .defects empty, so the only record of the problem is the generic 'invalid address in address-list' (or 'invalid mailbox in mailbox-list') defect on the enclosing token list, which does not say which character was misplaced. Attach an InvalidHeaderDefect naming the character to the terminal itself. It is picked up by all_defects of every enclosing token, so defect lists for malformed address headers gain one entry per misplaced character; the three tests that spell those lists out are updated accordingly. --- Lib/email/_header_value_parser.py | 6 ++-- .../test_email/test__header_value_parser.py | 29 ++++++++++++++++++- ...6-09-25-10-46-12.gh-issue-66889.EtcdVD.rst | 7 +++++ 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-25-10-46-12.gh-issue-66889.EtcdVD.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 7c03fb49b3b5e7..92adfd043f3efb 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1859,8 +1859,10 @@ def get_invalid_mailbox(value, endchars): invalid_mailbox = InvalidMailbox() while value and value[0] not in endchars: if value[0] in PHRASE_ENDS: - invalid_mailbox.append(ValueTerminal(value[0], - 'misplaced-special')) + special = ValueTerminal(value[0], 'misplaced-special') + special.defects.append(errors.InvalidHeaderDefect( + "misplaced special character {!r}".format(value[0]))) + invalid_mailbox.append(special) value = value[1:] else: token, value = get_phrase(value) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 9d9fe418ee4d06..d8608cf4d0e7b5 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2158,6 +2158,9 @@ def test_get_mailbox_list_unparseable_mailbox_null(self): ' "Fred Flintstone" '), [errors.InvalidHeaderDefect, # the 'extra' text after the local part errors.InvalidHeaderDefect, # the local part with no angle-addr + errors.InvalidHeaderDefect, # misplaced '[' in the extra text + errors.InvalidHeaderDefect, # misplaced ']' in the extra text + errors.InvalidHeaderDefect, # misplaced '@' in the extra text errors.ObsoleteHeaderDefect, # period in extra text (example.com) errors.ObsoleteHeaderDefect], # (bird) in valid address. '') @@ -2186,7 +2189,9 @@ def test_get_mailbox_list_junk_after_valid_address(self): ' "Fred Flintstone" '), ('"Roy A. Bear" @@,' ' "Fred Flintstone" '), - [errors.InvalidHeaderDefect], + [errors.InvalidHeaderDefect, # the junk after the valid address + errors.InvalidHeaderDefect, # first misplaced '@' in the junk + errors.InvalidHeaderDefect], # second misplaced '@' in the junk '') self.assertEqual(len(mailbox_list.mailboxes), 1) self.assertEqual(len(mailbox_list.all_mailboxes), 2) @@ -2688,6 +2693,8 @@ def test_get_address_list_mailboxes_invalid_addresses(self): [errors.InvalidHeaderDefect, # invalid address in list errors.InvalidHeaderDefect, # 'Foo x' local part invalid. errors.InvalidHeaderDefect, # Missing . in 'Foo x' local part + errors.InvalidHeaderDefect, # misplaced '[' after the addr-spec + errors.InvalidHeaderDefect, # misplaced ']' after the addr-spec errors.ObsoleteHeaderDefect, # period in 'Is.' disp-name phrase errors.InvalidHeaderDefect, # no domain part in addr-spec errors.ObsoleteHeaderDefect], # addr-spec has comment in it @@ -2712,6 +2719,26 @@ def test_get_address_list_mailboxes_invalid_addresses(self): address_list.addresses[3].all_mailboxes[0].display_name, "Nobody Is. Special") + def test_get_address_list_misplaced_special_has_defect(self): + address_list = self._test_get_x(parser.get_address_list, + 'abc@xyz.c:om', + 'abc@xyz.c:om', + 'abc@xyz.c:om', + [errors.InvalidHeaderDefect, # invalid address in list + errors.InvalidHeaderDefect], # the misplaced ':' + '') + invalid_mailbox = address_list.addresses[1].all_mailboxes[0] + self.assertEqual(invalid_mailbox.token_type, 'invalid-mailbox') + misplaced = invalid_mailbox[0] + self.assertEqual(misplaced.token_type, 'misplaced-special') + self.assertEqual(misplaced, ':') + # The defect is attached to the misplaced-special token itself, so the + # offending character can be located in the parse tree. + self.assertDefectsEqual(misplaced.defects, + [errors.InvalidHeaderDefect]) + self.assertEqual(str(misplaced.defects[0]), + "misplaced special character ':'") + def test_get_address_list_group_empty(self): address_list = self._test_get_x(parser.get_address_list, 'Monty Python: ;', diff --git a/Misc/NEWS.d/next/Library/2026-09-25-10-46-12.gh-issue-66889.EtcdVD.rst b/Misc/NEWS.d/next/Library/2026-09-25-10-46-12.gh-issue-66889.EtcdVD.rst new file mode 100644 index 00000000000000..e83e1034fcb37b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-25-10-46-12.gh-issue-66889.EtcdVD.rst @@ -0,0 +1,7 @@ +The :mod:`email` header parser now attaches an ``InvalidHeaderDefect`` to the +``misplaced-special`` token it creates for a stray special character in an +invalid mailbox, instead of reporting the problem only on the enclosing +address list or mailbox list. As a result, +:attr:`~email.headerregistry.BaseHeader.defects` for a malformed address +header such as ``To: abc@xyz.c:om`` gains one entry per misplaced character. +Patch by Dmitry Voropaev. From ab94ca350da9259df895f920d39268c9a2aad0d5 Mon Sep 17 00:00:00 2001 From: Dmitry Voropaev Date: Fri, 25 Sep 2026 11:12:34 +0300 Subject: [PATCH 2/2] gh-66889: Collect parse tree defects in linear time TokenList.all_defects built its result with sum(), which rebuilds the whole accumulated list once per child token, so gathering the defects of a parse tree costs time quadratic in the number of defects it holds. Until now that was mostly hidden, because the containers that collect many defects are rare; attaching a defect to every misplaced special character makes an ordinary malformed address header hit it. Parsing 'To: ' + '@'*32000 goes from 0.0154 to 1.08 seconds, and the pre-existing worst case, a long run of backslashes in an obs-local-part, already takes 4.45 seconds at the same size on main. Extend one list instead. Terminal.all_defects already returns a fresh copy, so returning a copy from an empty TokenList rather than the .defects list itself also makes the two agree. --- Lib/email/_header_value_parser.py | 5 ++++- .../test_email/test__header_value_parser.py | 19 +++++++++++++++++++ ...6-09-25-11-02-30.gh-issue-66889.Kq7nXb.rst | 4 ++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-25-11-02-30.gh-issue-66889.Kq7nXb.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 92adfd043f3efb..876c9ab70717f5 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -152,7 +152,10 @@ def value(self): @property def all_defects(self): - return sum((x.all_defects for x in self), self.defects) + defects = list(self.defects) + for x in self: + defects.extend(x.all_defects) + return defects def startswith_fws(self): return self[0].startswith_fws() diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index d8608cf4d0e7b5..0590353caed508 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -16,6 +16,25 @@ def test_EWWhiteSpaceTerminal(self): self.assertEqual(x.value, '') self.assertEqual(x.token_type, 'fws') + def test_all_defects_collects_subtree_in_order(self): + leaf = parser.ValueTerminal('x', 'atext') + leaf.defects.append(errors.InvalidHeaderDefect('leaf')) + inner = parser.TokenList([leaf]) + inner.defects.append(errors.InvalidHeaderDefect('inner')) + outer = parser.TokenList([inner]) + outer.defects.append(errors.InvalidHeaderDefect('outer')) + self.assertEqual([str(d) for d in outer.all_defects], + ['outer', 'inner', 'leaf']) + + def test_all_defects_is_a_copy(self): + tl = parser.TokenList() + tl.defects.append(errors.InvalidHeaderDefect('only')) + defects = tl.all_defects + self.assertEqual([str(d) for d in defects], ['only']) + self.assertIsNot(defects, tl.defects) + defects.append(errors.InvalidHeaderDefect('added')) + self.assertEqual(len(tl.defects), 1) + class TestParserMixin: diff --git a/Misc/NEWS.d/next/Library/2026-09-25-11-02-30.gh-issue-66889.Kq7nXb.rst b/Misc/NEWS.d/next/Library/2026-09-25-11-02-30.gh-issue-66889.Kq7nXb.rst new file mode 100644 index 00000000000000..5e0affc8982a5e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-25-11-02-30.gh-issue-66889.Kq7nXb.rst @@ -0,0 +1,4 @@ +Parsing a malformed :mod:`email` address header no longer takes time +quadratic in the number of defects the parser reports. Collecting the +defects of a parse tree copied the accumulated list once per node; it now +appends to a single list. Patch by Dmitry Voropaev.