From c515d374990ff0e0f91672d8fa0a61d8d20ab286 Mon Sep 17 00:00:00 2001
From: Dmitry Voropaev <dy.voropaev@gmail.com>
Date: Thu, 24 Sep 2026 23:00:10 +0300
Subject: [PATCH] gh-67486: Improve the error when detect_encoding() is given
 text

tokenize.detect_encoding() never checks what readline hands back, so a
readline returning str fails on whatever touches the line first. In the
common case that is first.startswith(BOM_UTF8), reporting "startswith
first arg must be str or a tuple of str, not bytes"; if the first line is
blank the failure moves into find_cookie() and becomes "cannot use a
bytes pattern on a string-like object". Neither mentions readline. Check
the line in read_or_stop() instead, and say both that readline has to
return bytes and where to go next: read the source as bytes, or use
generate_tokens() for text.

The exception type is unchanged, since str input raised TypeError before
and still does. The check rejects str rather than requiring bytes, so a
readline returning bytearray keeps working, and it sits after the
StopIteration handler so a readline that is already exhausted still
gives ('utf-8', []).
---
 Doc/library/tokenize.rst                      |  3 +-
 Lib/test/test_tokenize.py                     | 28 +++++++++++++++++++
 Lib/tokenize.py                               |  9 +++++-
 ...6-09-24-22-58-45.gh-issue-67486.Ke_NQN.rst |  4 +++
 4 files changed, 42 insertions(+), 2 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2026-09-24-22-58-45.gh-issue-67486.Ke_NQN.rst

diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst
index 2eea51734fde03c..b516d2a8617d915 100644
--- a/Doc/library/tokenize.rst
+++ b/Doc/library/tokenize.rst
@@ -109,7 +109,8 @@ function it uses to do this is available:
 
     It will call readline a maximum of twice, and return the encoding used
     (as a string) and a list of any lines (not decoded from bytes) it has read
-    in.
+    in. A :exc:`TypeError` is raised if readline returns a :class:`str`
+    instead of bytes.
 
     It detects the encoding from the presence of a UTF-8 BOM or an encoding
     cookie as specified in :pep:`263`. If both a BOM and a cookie are present,
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 53215eceeb8aed3..4a9b873ee49eb48 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1693,6 +1693,34 @@ def test_false_encoding(self):
         self.assertEqual(encoding, 'utf-8')
         self.assertEqual(consumed_lines, [b'print("#coding=fake")'])
 
+    def test_readline_returning_str(self):
+        # gh-67486: readline must return bytes here, and saying so is more
+        # helpful than whatever the first operation on the line happens to
+        # complain about.
+        expected = 'readline callable returning bytes, but it returned str'
+
+        readline = self.get_readline(('print(something)\n',))
+        with self.assertRaisesRegex(TypeError, expected):
+            tokenize.detect_encoding(readline)
+
+        # The second line is read only when the first one is blank.
+        readline = self.get_readline((b'\n', 'print(something)\n'))
+        with self.assertRaisesRegex(TypeError, expected):
+            tokenize.detect_encoding(readline)
+
+        # The reported case went through tokenize(), which calls
+        # detect_encoding() itself.
+        with self.assertRaisesRegex(TypeError, expected):
+            list(tokenize.tokenize(StringIO('print(something)\n').readline))
+
+        # Only str is rejected. bytearray is not a documented input, but it
+        # has always worked, so a check for bytes exactly would break it.
+        line = bytearray(b'print(something)\n')
+        readline = self.get_readline((line,))
+        encoding, consumed_lines = tokenize.detect_encoding(readline)
+        self.assertEqual(encoding, 'utf-8')
+        self.assertEqual(consumed_lines, [line])
+
     @support.thread_unsafe
     def test_open(self):
         filename = os_helper.TESTFN + '.py'
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 3545d92c4f5d7ff..6a78f57c280181d 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -364,6 +364,7 @@ def detect_encoding(readline):
 
     It will call readline a maximum of twice, and return the encoding used
     (as a string) and a list of any lines (left as bytes) it has read in.
+    A TypeError is raised if readline returns a str instead of bytes.
 
     It detects the encoding from the presence of a utf-8 bom or an encoding
     cookie as specified in pep-0263.  If both a bom and a cookie are present,
@@ -382,9 +383,15 @@ def detect_encoding(readline):
     default = 'utf-8'
     def read_or_stop():
         try:
-            return readline()
+            line = readline()
         except StopIteration:
             return b''
+        if isinstance(line, str):
+            raise TypeError("detect_encoding() requires a readline callable "
+                            "returning bytes, but it returned str; read the "
+                            "source as bytes, or use "
+                            "tokenize.generate_tokens() to tokenize text")
+        return line
 
     def check(line, encoding):
         # Check if the line matches the encoding.
diff --git a/Misc/NEWS.d/next/Library/2026-09-24-22-58-45.gh-issue-67486.Ke_NQN.rst b/Misc/NEWS.d/next/Library/2026-09-24-22-58-45.gh-issue-67486.Ke_NQN.rst
new file mode 100644
index 000000000000000..68643b0745903cd
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-09-24-22-58-45.gh-issue-67486.Ke_NQN.rst
@@ -0,0 +1,4 @@
+When *readline* returns a string instead of bytes,
+:func:`tokenize.detect_encoding` now raises a :exc:`TypeError` that says so.
+Previously the failure came later, from whichever operation touched the line
+first, and did not mention *readline*. Patch by Dmitry Voropaev.