gh-94823: Improve coverage in tokenizer.c:valid_utf8

When loading a source file from disk, there is a separate UTF-8 validator distinct from the one in `unicode_decode_utf8`. This exercises that code path with the same set of invalid inputs as we use for testing the "other" UTF-8 decoder.
miss-islington · Aug 16, 2022 · Jul 14, 2022 · Jul 20, 2022 · Aug 12, 2022 · Aug 16, 2022
commit e52f328e5920258060adec0337ffca77937e6b14
diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
@@ -224,6 +224,64 @@ def test_crcrcrlf2(self):
         out = self.check_script_output(src, br"'\n\n\n'")
 
 
+class UTF8ValidatorTest(unittest.TestCase):
+    @unittest.skipIf(sys.platform.startswith("win"),
+                     "Times out on Windows")
+    def test_invalid_utf8(self):
+        # This is port of test_utf8_decode_invalid_sequences in test_unicode.py
+        # to exercise the separate utf8 validator in tokenize.c used when
+        # reading source files.
+
+        # Each example is put inside a string at the top of the file so
+        # it's an otherwise valid Python source file.
+        template = b'"%s"\n'
+
+        with tempfile.TemporaryDirectory() as tmpd:
+            fn = os.path.join(tmpd, 'test.py')
+
+            def check(content):
+                with open(fn, 'wb') as fp:
+                    fp.write(template % content)
+                script_helper.assert_python_failure(fn)
+
+            # continuation bytes in a sequence of 2, 3, or 4 bytes
+            continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0, 7)]
+            # start bytes of a 2-byte sequence equivalent to code points < 0x7F
+            invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)]
+            # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF
+            invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)]
+            invalid_start_bytes = (
+                continuation_bytes + invalid_2B_seq_start_bytes +
+                invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)]
+            )
+
+            for byte in invalid_start_bytes:
+                check(byte)
+
+            for sb in invalid_2B_seq_start_bytes:
+                for cb in continuation_bytes:
+                    check(sb + cb)
+
+            for sb in invalid_4B_seq_start_bytes:
+                for cb1 in continuation_bytes[:3]:
+                    for cb3 in continuation_bytes[:3]:
+                        check(sb+cb1+b'\x80'+cb3)
+
+            for cb in [bytes([x]) for x in range(0x80, 0xA0, 5)]:
+                check(b'\xE0'+cb+b'\x80')
+                check(b'\xE0'+cb+b'\xBF')
+                # surrogates
+            for cb in [bytes([x]) for x in range(0xA0, 0xC0, 5)]:
+                check(b'\xED'+cb+b'\x80')
+                check(b'\xED'+cb+b'\xBF')
+            for cb in [bytes([x]) for x in range(0x80, 0x90, 5)]:
+                check(b'\xF0'+cb+b'\x80\x80')
+                check(b'\xF0'+cb+b'\xBF\xBF')
+            for cb in [bytes([x]) for x in range(0x90, 0xC0, 5)]:
+                check(b'\xF4'+cb+b'\x80\x80')
+                check(b'\xF4'+cb+b'\xBF\xBF')
+
+
 class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):
 
     def check_script_output(self, src, expected):