diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 63006266bcd039e..72c80b8bfbfecc5 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self): f"{prefix[-1]}-string: expecting '}}' to close '{{' " f"on line {lineno}") + def test_unclosed_replacement_field_quote_line(self): + for prefix in ('f', 't', 'rf', 'rt'): + for quote in ('"', "'"): + triple = quote * 3 + for suffix in ('', '\nx'): + source = prefix + triple + '{1' + triple + suffix + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + self.assertEqual( + cm.exception.msg, + f"{prefix[-1]}-string: expecting '}}'") + self.assertEqual(cm.exception.lineno, 1) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 21e603c561d73c0..3525dee84493acc 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -187,16 +187,19 @@ def read_until(marker, start=0): def test_lexer_buffer_realloc_with_null_start(self): # gh-144759: NULL pointer arithmetic when the lexer buffer grows # while parsing long input. - long_value = "a" * 2000 + long_value = "é漢" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' print(x) """) - p = spawn_repl() - p.stdin.write(user_input) - output = kill_python(p) - self.assertEqual(p.returncode, 0) - self.assertIn(long_value, output) + for newline in ("\n", "\r\n"): + with self.subTest(newline=newline): + p = spawn_repl(encoding="utf-8") + # Bypass Windows text-mode translation of CRLF to CRCRLF. + p.stdin.buffer.write(user_input.replace("\n", newline).encode("utf-8")) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(long_value, output) @cpython_only def test_multiline_fstring_source_reallocation(self): diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index ec98e609c4e98f9..3c29002025575e1 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -29,6 +29,30 @@ def test_compilestring(self): exec(c, d) self.assertEqual(d['u'], '\xf3') + def test_compilestring_line_endings(self): + for newline in ('\n', '\r', '\r\n', '\r\r\n'): + expected = 'é\n\ntext' if newline == '\r\r\n' else 'é\ntext' + for suffix in ('', newline): + for mode in ('exec', 'eval'): + source = f"'''é{newline}text'''{suffix}" + if mode == 'exec': + source = 'value = ' + source + for encoding in (None, 'utf-8', 'latin-1'): + with self.subTest(newline=newline, suffix=suffix, + mode=mode, encoding=encoding): + input = source + if encoding is not None: + input = (f'# coding: {encoding}{newline}' + + source).encode(encoding) + code = compile(input, '', mode) + if mode == 'exec': + namespace = {} + exec(code, namespace) + value = namespace['value'] + else: + value = eval(code) + self.assertEqual(value, expected) + def test_issue2301(self): try: compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec") @@ -135,6 +159,11 @@ def test_stateful_file_decoder_spans_lines(self): ) self._assert_python_file_ok(source) + @support.requires_subprocess() + def test_stateful_file_decoder_preserves_buffered_text(self): + source = b"# coding: hz\nx~\ny = 1\nassert xy == 1\n" + self._assert_python_file_ok(source) + @support.requires_subprocess() def test_stateful_file_decoder_finalizes_before_implicit_newline(self): source = b"# coding: hz\n# ~{1dA?" diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index cb8f58511aab877..9fa294aa3618845 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -8,6 +8,7 @@ import token import tokenize import unittest +import weakref from io import BytesIO, StringIO from textwrap import dedent from unittest import TestCase, mock @@ -2268,6 +2269,18 @@ def check_tokenize(self, s, expected): ) self.assertEqual(result, expected.rstrip().splitlines()) + def test_readline_reference_cycle(self): + class Readline: + def __call__(self): + return "" + + readline = Readline() + readline.iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + ref = weakref.ref(readline) + del readline + support.gc_collect() + self.assertIsNone(ref()) + def test_encoding(self): def readline(encoding): yield "1+1".encode(encoding) @@ -2341,6 +2354,20 @@ def test_utf8_decoder_spans_readline_calls(self): tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""), ]) + def test_utf8_decoder_spans_many_readline_calls(self): + for prefix in (b"", b"previous\n"): + with self.subTest(prefix=prefix): + chunks = ([prefix + b"x\xc3"] + [b"\xa9\xc3"] * 100 + + [b"\xa9\n", b"z\xc3", b"\xa9\n", b""]) + source = b"".join(chunks) + expected = list(_tokenize.TokenizerIter( + BytesIO(source).readline, encoding="utf-8", extra_tokens=True + )) + tokens = list(_tokenize.TokenizerIter( + iter(chunks).__next__, encoding="utf-8", extra_tokens=True + )) + self.assertEqual(tokens, expected) + def test_utf8_decoder_replaces_incomplete_input_at_eof(self): expected = [ tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"), @@ -2396,6 +2423,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self): ) self.assertEqual(readline.call_count, 2) + def test_readline_memory_error_in_string(self): + readline = mock.Mock(side_effect=['"""first\n', MemoryError]) + iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + with self.assertRaises(MemoryError): + next(iterator) + def test_readline_callback_is_not_read_ahead(self): readline = mock.Mock(side_effect=["x\n", "y\n", ""]) iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index d827ac271085bdd..832a1e283e0c279 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -338,6 +338,16 @@ def test_ignores(self): tree = self.classic_parse(ignores) self.assertEqual(tree.type_ignores, []) + def test_many_ignores(self): + comment_count = 25 + tags = [f"[tag_{index}]" for index in range(comment_count)] + source = "".join(f"pass # type: ignore{tag}\n" for tag in tags) + for tree in self.parse_all(source): + self.assertEqual( + [(item.lineno, item.tag) for item in tree.type_ignores], + list(enumerate(tags, start=1)), + ) + def test_longargs(self): for tree in self.parse_all(longargs, minver=8): for t in tree.body: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst new file mode 100644 index 000000000000000..7bd6b023fb32da5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -0,0 +1,4 @@ +Correct missing-brace error messages for multiline f-strings and t-strings. +Improve tokenization of UTF-8 input split across many readline calls. +Preserve source order when reading files with stateful encodings. +Preserve input errors when scanning string literals. diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 1f89c12f223c7d9..0c2bdee2e803b22 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -38,6 +38,12 @@ static PyObject * test_tokenizer_source(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + const char multiple_lines[] = "a\nb\n"; + const char first_line[] = "alpha\n"; + const char second_line[] = "\xce\xb2\n"; + const char expected[] = "alpha\n\xce\xb2\n"; + const char tail[] = "tail"; + const char terminated_line[] = "x\n"; _PyTok_SourceText source; _PyTok_SourceInit(&source); @@ -46,23 +52,19 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), } if (check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + _PyTok_SourceAppendLine(&source, "", 0) < 0, "accepted empty source line") < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, + _PyTok_SourceAppendLine( + &source, multiple_lines, sizeof(multiple_lines) - 1) < 0, "accepted multiple source lines") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0 || check(_PyTok_SourceAppendLine( - &source, "alpha\n", 6, 0) == 0, + &source, first_line, sizeof(first_line) - 1) == 0, "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( - &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 1) && - _PyTok_SourceLineIsImplicit(&source, 2), - "wrong implicit newline flags") < 0) { + &source, second_line, sizeof(second_line) - 1) == + (Py_ssize_t)sizeof(first_line) - 1, + "wrong second source offset") < 0) { goto error; } @@ -74,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - if (check(source.len == 9 && - memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 && + memcmp(source.bytes, expected, sizeof(expected)) == 0, "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || + if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, + _PyTok_SourceAppendLine( + &source, terminated_line, sizeof(terminated_line) - 1) < 0, "appended after unterminated source line") < 0) { goto error; } @@ -104,27 +107,35 @@ static PyObject * test_tokenizer_source_discard(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + enum { LINE_COUNT = 2 }; + const char first_line[] = "x\n"; + const char second_line[] = "y\n"; + const char tail[] = "tail"; + const char final_line[] = "z\n"; + const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1); + const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1); + const _PyTok_Off discarded_len = first_batch_len + second_batch_len; _PyTok_SourceText source; _PyTok_SourceInit(&source); - for (int i = 0; i < 260; i++) { - if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) { goto error; } } char *bytes = source.bytes; _PyTok_Off capacity = source.cap; _PyTok_SourceDiscard(&source); - if (check(source.base_offset == 520 && source.len == 0 && + if (check(source.base_offset == first_batch_len && source.len == 0 && source.nlines == 0 && source.bytes == bytes && source.cap == capacity && source.bytes[0] == '\0', "discard did not preserve source allocation") < 0) { goto error; } - for (int i = 0; i < 260; i++) { - if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i, - "wrong source offset after discard") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, i + 1), - "discard preserved implicit newline flag") < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (check(_PyTok_SourceAppendLine( + &source, second_line, sizeof(second_line) - 1) == + first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i, + "wrong source offset after discard") < 0) { goto error; } } @@ -133,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040, + if (check(_PyTok_SourceAppendLine( + &source, tail, sizeof(tail) - 1) == discarded_len, "wrong source offset after repeated discard") < 0) { goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044, + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) == + discarded_len + (Py_ssize_t)sizeof(tail) - 1, "cannot append after discarding unterminated line") < 0) { goto error; } _PyTok_SourceDiscard(&source); source.base_offset = PY_SSIZE_T_MAX - 1; - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 && + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) < 0 && PyErr_ExceptionMatches(PyExc_MemoryError), "accepted overflowing logical source offset") < 0) { goto error; diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 4652103107367ef..a4fdb9484abd8de 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -314,7 +314,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token // Handle valid f or t string creation: if (saw_f || saw_t) { - return _PyLexer_scan_fstring_start(tok, token, c); + ftstring_kind kind = saw_t + ? (saw_r ? RAW_TSTRING : TSTRING) + : (saw_r ? RAW_FSTRING : FSTRING); + return _PyLexer_scan_fstring_start(tok, token, c, kind); } return _PyLexer_scan_string(tok, token, c); } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index f0d9576d95f8ff3..652512a3e88c22d 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -66,7 +66,8 @@ int _PyLexer_close_ftstring_expr( void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); -int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); +int _PyLexer_scan_fstring_start( + struct tok_state *, struct token *, int, ftstring_kind); int _PyLexer_scan_string(struct tok_state *, struct token *, int); int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *); int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index a48cedf270362ab..99e45578b6ca16e 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -13,7 +13,7 @@ string_error_token(struct tok_state *tok, struct token *token, { tok->diagnostic = (_PyTokenizer_Diagnostic){ .location = {location.lineno, location.byte_col + 1}, - .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp), + .text_span = {start - location.byte_col, tok->inp}, }; int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1); token->start_loc = location; @@ -83,18 +83,17 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state *state, PyObject *res; if (comments != NULL && comments->count > 0) { Py_ssize_t stripped_size = expr_len; - _PyTok_Off previous_end = state->expr_span.start; Py_ssize_t comment_count = 0; for (Py_ssize_t i = 0; i < comments->count; i++) { _PyTok_Span comment = comments->spans[i]; assert(_PyTok_SpanIsValid(comment)); - assert(comment.start >= previous_end); + assert(comment.start >= (i == 0 ? state->expr_span.start + : comments->spans[i - 1].end)); if (comment.start >= state->expr_span.end) { break; } assert(comment.end <= state->expr_span.end); stripped_size -= comment.end - comment.start; - previous_end = comment.end; comment_count++; } char *stripped = PyMem_Malloc((size_t)stripped_size); @@ -254,7 +253,8 @@ _PyLexer_check_string_prefixes(struct tok_state *tok, } int -_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) +_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, + int c, ftstring_kind kind) { _PyTok_Off p_start = -1; _PyTok_Off p_end = -1; @@ -293,30 +293,9 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) state->start_loc = tok->start_loc; state->expr_span = (_PyTok_Span){-1, -1}; - int raw = 0; - int tstring = 0; - switch (*_PyLexer_BufferPointer(tok, tok->start)) { - case 'T': - case 't': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - tstring = 1; - break; - case 'F': - case 'f': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - break; - case 'R': - case 'r': - raw = 1; - tstring = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 't'; - break; - default: - Py_UNREACHABLE(); - } - state->kind = tstring - ? (raw ? RAW_TSTRING : TSTRING) - : (raw ? RAW_FSTRING : FSTRING); - return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); + state->kind = kind; + return _PyLexer_IsTString(kind) + ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); } int @@ -355,6 +334,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { + if (tok_failed(tok)) { + return MAKE_TOKEN(ERRORTOKEN); + } int end_lineno = tok->lineno; _PyTok_Loc location = tok->start_loc; const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col; @@ -370,7 +352,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) assert(level >= 0 && level < tok->level); assert(tok->parenstack[level] == '{'); int lineno = tok->parenlinenostack[level]; - if (lineno != tok->lineno) { + if (lineno != location.lineno) { _PyTokenizer_syntaxerror_at( tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}' to close '{' on line %d", diff --git a/Parser/parser.c b/Parser/parser.c index 896c089e3360035..979907f4873b9dd 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27350,14 +27350,8 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // NEWLINE if (p->error_indicator) { @@ -27372,16 +27366,13 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27392,9 +27383,9 @@ _loop0_1_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -27417,14 +27408,8 @@ _loop1_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // statement if (p->error_indicator) { @@ -27439,16 +27424,13 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27464,9 +27446,9 @@ _loop1_2_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -27489,14 +27471,8 @@ _loop0_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ';' simple_stmt if (p->error_indicator) { @@ -27520,16 +27496,13 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -27540,9 +27513,9 @@ _loop0_3_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28043,14 +28016,8 @@ _loop1_12_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -28065,16 +28032,13 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28090,9 +28054,9 @@ _loop1_12_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28115,14 +28079,8 @@ _loop0_13_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' NAME if (p->error_indicator) { @@ -28146,16 +28104,13 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28166,9 +28121,9 @@ _loop0_13_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28335,14 +28290,8 @@ _loop0_17_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28357,16 +28306,13 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28377,9 +28323,9 @@ _loop0_17_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28402,14 +28348,8 @@ _loop1_18_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28424,16 +28364,13 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28449,9 +28386,9 @@ _loop1_18_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28474,14 +28411,8 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' import_from_as_name if (p->error_indicator) { @@ -28505,16 +28436,13 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28525,9 +28453,9 @@ _loop0_19_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28637,14 +28565,8 @@ _loop0_22_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_as_name if (p->error_indicator) { @@ -28668,16 +28590,13 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28688,9 +28607,9 @@ _loop0_22_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28754,14 +28673,8 @@ _loop1_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('@' named_expression NEWLINE) if (p->error_indicator) { @@ -28776,16 +28689,13 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28801,9 +28711,9 @@ _loop1_24_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28921,14 +28831,8 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -28943,16 +28847,13 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -28963,9 +28864,9 @@ _loop0_27_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -28988,14 +28889,8 @@ _loop0_28_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29010,16 +28905,13 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29030,9 +28922,9 @@ _loop0_28_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29055,14 +28947,8 @@ _loop1_29_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -29077,16 +28963,13 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29102,9 +28985,9 @@ _loop1_29_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29127,14 +29010,8 @@ _loop1_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29149,16 +29026,13 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29174,9 +29048,9 @@ _loop1_30_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29199,14 +29073,8 @@ _loop0_31_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29221,16 +29089,13 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29241,9 +29106,9 @@ _loop0_31_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29266,14 +29131,8 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29288,16 +29147,13 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29313,9 +29169,9 @@ _loop1_32_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29338,14 +29194,8 @@ _loop0_33_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' with_item if (p->error_indicator) { @@ -29369,16 +29219,13 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29389,9 +29236,9 @@ _loop0_33_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29531,14 +29378,8 @@ _loop1_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_block if (p->error_indicator) { @@ -29553,16 +29394,13 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29578,9 +29416,9 @@ _loop1_36_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29603,14 +29441,8 @@ _loop1_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_star_block if (p->error_indicator) { @@ -29625,16 +29457,13 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29650,9 +29479,9 @@ _loop1_37_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29675,14 +29504,8 @@ _loop1_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // case_block if (p->error_indicator) { @@ -29697,16 +29520,13 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29722,9 +29542,9 @@ _loop1_38_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -29747,14 +29567,8 @@ _loop0_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // '|' closed_pattern if (p->error_indicator) { @@ -29778,16 +29592,13 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -29798,9 +29609,9 @@ _loop0_39_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30073,14 +29884,8 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' maybe_star_pattern if (p->error_indicator) { @@ -30104,16 +29909,13 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30124,9 +29926,9 @@ _loop0_44_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30190,14 +29992,8 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' key_value_pattern if (p->error_indicator) { @@ -30221,16 +30017,13 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30241,9 +30034,9 @@ _loop0_46_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30364,14 +30157,8 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' pattern if (p->error_indicator) { @@ -30395,16 +30182,13 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30415,9 +30199,9 @@ _loop0_49_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30481,14 +30265,8 @@ _loop0_51_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' keyword_pattern if (p->error_indicator) { @@ -30512,16 +30290,13 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30532,9 +30307,9 @@ _loop0_51_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30598,14 +30373,8 @@ _loop0_53_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' type_param if (p->error_indicator) { @@ -30629,16 +30398,13 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30649,9 +30415,9 @@ _loop0_53_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30715,14 +30481,8 @@ _loop1_55_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' expression) if (p->error_indicator) { @@ -30737,16 +30497,13 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30762,9 +30519,9 @@ _loop1_55_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30787,14 +30544,8 @@ _loop1_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_expression) if (p->error_indicator) { @@ -30809,16 +30560,13 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30834,9 +30582,9 @@ _loop1_56_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30859,14 +30607,8 @@ _loop0_57_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' star_named_expression if (p->error_indicator) { @@ -30890,16 +30632,13 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -30910,9 +30649,9 @@ _loop0_57_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -30976,14 +30715,8 @@ _loop1_59_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('or' conjunction) if (p->error_indicator) { @@ -30998,16 +30731,13 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31023,9 +30753,9 @@ _loop1_59_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31048,14 +30778,8 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('and' inversion) if (p->error_indicator) { @@ -31070,16 +30794,13 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31095,9 +30816,9 @@ _loop1_60_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31120,14 +30841,8 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // compare_op_bitwise_or_pair if (p->error_indicator) { @@ -31142,16 +30857,13 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31167,9 +30879,9 @@ _loop1_61_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31235,14 +30947,8 @@ _loop0_63_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (slice | starred_expression) if (p->error_indicator) { @@ -31266,16 +30972,13 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31286,9 +30989,9 @@ _loop0_63_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31683,14 +31386,8 @@ _loop0_70_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31705,16 +31402,13 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31725,9 +31419,9 @@ _loop0_70_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31750,14 +31444,8 @@ _loop0_71_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31772,16 +31460,13 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31792,9 +31477,9 @@ _loop0_71_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31817,14 +31502,8 @@ _loop1_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31839,16 +31518,13 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31864,9 +31540,9 @@ _loop1_72_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31889,14 +31565,8 @@ _loop1_73_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31911,16 +31581,13 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -31936,9 +31603,9 @@ _loop1_73_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -31961,14 +31628,8 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -31983,16 +31644,13 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32003,9 +31661,9 @@ _loop0_74_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32028,14 +31686,8 @@ _loop1_75_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -32050,16 +31702,13 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32075,9 +31724,9 @@ _loop1_75_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32100,14 +31749,8 @@ _loop0_76_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_format_spec if (p->error_indicator) { @@ -32122,16 +31765,13 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32142,9 +31782,9 @@ _loop0_76_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32167,14 +31807,8 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_middle if (p->error_indicator) { @@ -32189,16 +31823,13 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32209,9 +31840,9 @@ _loop0_77_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32234,14 +31865,8 @@ _loop0_78_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_format_spec if (p->error_indicator) { @@ -32256,16 +31881,13 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32276,9 +31898,9 @@ _loop0_78_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32301,14 +31923,8 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_middle if (p->error_indicator) { @@ -32323,16 +31939,13 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32343,9 +31956,9 @@ _loop0_79_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32368,14 +31981,8 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (fstring | string) if (p->error_indicator) { @@ -32390,16 +31997,13 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32415,9 +32019,9 @@ _loop1_80_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32440,14 +32044,8 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring if (p->error_indicator) { @@ -32462,16 +32060,13 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32487,9 +32082,9 @@ _loop1_81_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32512,14 +32107,8 @@ _loop0_82_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' display_item if (p->error_indicator) { @@ -32543,16 +32132,13 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32563,9 +32149,9 @@ _loop0_82_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32678,14 +32264,8 @@ _loop0_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' double_starred_kvpair if (p->error_indicator) { @@ -32709,16 +32289,13 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32729,9 +32306,9 @@ _loop0_85_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32795,14 +32372,8 @@ _loop1_87_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // for_if_clause if (p->error_indicator) { @@ -32817,16 +32388,13 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32842,9 +32410,9 @@ _loop1_87_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -32867,14 +32435,8 @@ _loop0_88_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('if' disjunction) if (p->error_indicator) { @@ -32889,16 +32451,13 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -32909,9 +32468,9 @@ _loop0_88_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33012,14 +32571,8 @@ _loop0_90_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') if (p->error_indicator) { @@ -33043,16 +32596,13 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33063,9 +32613,9 @@ _loop0_90_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33176,14 +32726,8 @@ _loop0_93_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_starred if (p->error_indicator) { @@ -33207,16 +32751,13 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33227,9 +32768,9 @@ _loop0_93_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33339,14 +32880,8 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_double_starred if (p->error_indicator) { @@ -33370,16 +32905,13 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33390,9 +32922,9 @@ _loop0_96_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33456,14 +32988,8 @@ _loop0_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_target) if (p->error_indicator) { @@ -33478,16 +33004,13 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33498,9 +33021,9 @@ _loop0_98_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33639,14 +33162,8 @@ _loop0_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' del_target if (p->error_indicator) { @@ -33670,16 +33187,13 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33690,9 +33204,9 @@ _loop0_101_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33756,14 +33270,8 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' expression if (p->error_indicator) { @@ -33787,16 +33295,13 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -33807,9 +33312,9 @@ _loop0_103_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -33973,14 +33478,8 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression !'=') if (p->error_indicator) { @@ -34004,16 +33503,13 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34024,9 +33520,9 @@ _loop0_107_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34289,14 +33785,8 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (!STRING expression_without_invalid) if (p->error_indicator) { @@ -34311,16 +33801,13 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34336,9 +33823,9 @@ _loop1_113_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34744,14 +34231,8 @@ _loop0_119_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // star_named_expressions if (p->error_indicator) { @@ -34766,16 +34247,13 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34786,9 +34264,9 @@ _loop0_119_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -34811,14 +34289,8 @@ _loop0_120_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -34833,16 +34305,13 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -34853,9 +34322,9 @@ _loop0_120_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35356,14 +34825,8 @@ _loop0_129_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' lambda_param if (p->error_indicator) { @@ -35387,16 +34850,13 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35407,9 +34867,9 @@ _loop0_129_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35692,14 +35152,8 @@ _loop0_135_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_name if (p->error_indicator) { @@ -35723,16 +35177,13 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35743,9 +35194,9 @@ _loop0_135_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35850,14 +35301,8 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expression ['as' star_target]) if (p->error_indicator) { @@ -35881,16 +35326,13 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -35901,9 +35343,9 @@ _loop0_138_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -35967,14 +35409,8 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expressions ['as' star_target]) if (p->error_indicator) { @@ -35998,16 +35434,13 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -36018,9 +35451,9 @@ _loop0_140_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -36141,14 +35574,8 @@ _loop0_143_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // block if (p->error_indicator) { @@ -36163,16 +35590,13 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -36183,9 +35607,9 @@ _loop0_143_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } @@ -37794,14 +37218,8 @@ _loop0_172_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' bitwise_or) if (p->error_indicator) { @@ -37816,16 +37234,13 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); - if (!_new_children) { - PyMem_Free(_children); + if (_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0) { p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } - _children = _new_children; } _children[_n++] = _res; _mark = p->mark; @@ -37836,9 +37251,9 @@ _loop0_172_rule(Parser *p) } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); if (!_seq) { - PyMem_Free(_children); p->error_indicator = 1; PyErr_NoMemory(); + PyMem_Free(_children); p->level--; return NULL; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 7b228ca874bc0ba..4ca42a407bcde80 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -13,6 +13,8 @@ #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. #define IDENTIFIER_CACHE_MAX_PROBES 8 +#define PEGEN_ARRAY_GROWTH_FACTOR 2 +#define TYPE_IGNORE_INITIAL_CAPACITY 10 struct _identifier_cache_entry { const char *key; // Borrowed from arena-owned token bytes. @@ -36,6 +38,9 @@ Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset) { const unsigned char *data = (const unsigned char*)PyUnicode_AsUTF8(line); + if (data == NULL) { + return -1; + } Py_ssize_t len = 0; while (col_offset < end_col_offset) { @@ -144,21 +149,37 @@ init_normalization(Parser *p) return 1; } -static int -growable_comment_array_init(growable_comment_array *arr, size_t initial_size) { - assert(initial_size > 0); - arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items)); - arr->size = initial_size; - arr->num_items = 0; - - return arr->items != NULL; +int +_PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity) +{ + if ((size_t)*capacity > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(**buffer))) { + return -1; + } + Py_ssize_t new_capacity = *capacity == 0 + ? 1 : *capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **new_buffer = PyMem_Realloc(*buffer, new_capacity * sizeof(**buffer)); + if (new_buffer == NULL) { + return -1; + } + *buffer = new_buffer; + *capacity = new_capacity; + return 0; } static int -growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { +growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) +{ if (arr->num_items >= arr->size) { - size_t new_size = arr->size * 2; - void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items)); + if (arr->size > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*arr->items))) { + return 0; + } + size_t new_size = arr->size == 0 + ? TYPE_IGNORE_INITIAL_CAPACITY + : arr->size * PEGEN_ARRAY_GROWTH_FACTOR; + void *new_items_array = PyMem_Realloc( + arr->items, new_size * sizeof(*arr->items)); if (!new_items_array) { return 0; } @@ -173,8 +194,9 @@ growable_comment_array_add(growable_comment_array *arr, int lineno, char *commen } static void -growable_comment_array_deallocate(growable_comment_array *arr) { - for (unsigned i = 0; i < arr->num_items; i++) { +growable_comment_array_deallocate(growable_comment_array *arr) +{ + for (size_t i = 0; i < arr->num_items; i++) { PyMem_Free(arr->items[i].comment); } PyMem_Free(arr->items); @@ -274,12 +296,13 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to static int _resize_tokens_array(Parser *p) { - if (p->size > INT_MAX / 2 || - (size_t)p->size > PY_SSIZE_T_MAX / (2 * sizeof(*p->tokens))) { + if (p->size > INT_MAX / PEGEN_ARRAY_GROWTH_FACTOR || + (size_t)p->size > PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*p->tokens))) { PyErr_NoMemory(); return -1; } - int newsize = p->size * 2; + int newsize = p->size * PEGEN_ARRAY_GROWTH_FACTOR; Token **new_tokens = PyMem_Realloc(p->tokens, (size_t)newsize * sizeof(Token *)); if (new_tokens == NULL) { PyErr_NoMemory(); @@ -321,6 +344,7 @@ _PyPegen_fill_token(Parser *p) tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, new_token.end_loc.lineno, tag)) { + PyMem_Free(tag); PyErr_NoMemory(); goto error; } @@ -944,12 +968,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, PyMem_Free(p); return (Parser *) PyErr_NoMemory(); } - if (!growable_comment_array_init(&p->type_ignore_comments, 10)) { - PyMem_Free(p->tokens[0]); - PyMem_Free(p->tokens); - PyMem_Free(p); - return (Parser *) PyErr_NoMemory(); - } + p->type_ignore_comments = (growable_comment_array){0}; p->mark = 0; p->fill = 0; @@ -996,7 +1015,7 @@ _PyPegen_Parser_Free(Parser *p) Py_XDECREF(p->normalize); // Resizes allocate blocks starting at indices 1, 2, 4, and so on. PyMem_Free(p->tokens[0]); - for (int i = 1; i < p->size; i *= 2) { + for (int i = 1; i < p->size; i *= PEGEN_ARRAY_GROWTH_FACTOR) { PyMem_Free(p->tokens[i]); } PyMem_Free(p->tokens); diff --git a/Parser/pegen.h b/Parser/pegen.h index eb3f1d8a3041a7c..7454289ef6df03d 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -190,6 +190,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p); expr_ty _PyPegen_fstring_middle_token(Parser* p); Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); int _PyPegen_fill_token(Parser *p); +// Return -1 on failure, leaving the buffer and capacity unchanged. +int _PyPegen_grow_loop_buffer(void ***buffer, Py_ssize_t *capacity); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 74ab56c912ae1af..6997a1158e9310f 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -327,6 +327,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, } tmp = Py_BuildValue("(OnnNnn)", info.filename, lineno, col_number, error_line, end_lineno, end_col_number); + error_line = NULL; if (!tmp) { goto error; } diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 69c3bb371add960..220fed65ad4e450 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -8,6 +8,14 @@ #include "helpers.h" #include "../lexer/state.h" +#define UNICODE_BOM 0xFEFF +#define UTF16_LE_BOM "\xff\xfe" +#define UTF16_BE_BOM "\xfe\xff" +#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1)) +#define UTF8_BOM "\xef\xbb\xbf" +#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1)) +#define NORMALIZED_LINE_INITIAL_CAPACITY 256 + char * _PyTok_CopyBytes(const char *data, Py_ssize_t len) { @@ -60,9 +68,9 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return -1; } if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 && - PyUnicode_ReadChar(unicode, 0) == 0xFEFF) { - utf8 += 3; - utf8_len -= 3; + PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) { + utf8 += UTF8_BOM_LENGTH; + utf8_len -= UTF8_BOM_LENGTH; } chunk_release_data(chunk); chunk->owner = unicode; @@ -72,58 +80,32 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return 0; } -// The caller must provide len + 2 bytes: normalization can append a final -// newline and always writes a NUL terminator. -static void -normalize_newlines_into(char *result, const char *data, Py_ssize_t len, - int preserve_crlf, int add_final_newline, - Py_ssize_t *out_len, int *implicit_newline) -{ - Py_ssize_t write = 0; - if (preserve_crlf || memchr(data, '\r', len) == NULL) { - // No translation needed: copy verbatim. - memcpy(result, data, len); - write = len; - } - else { - for (Py_ssize_t read = 0; read < len; read++) { - char c = data[read]; - if (c == '\r') { - if (read + 1 < len && data[read + 1] == '\n') { - read++; - } - c = '\n'; - } - result[write++] = c; - } - } - int implicit = add_final_newline && write > 0 && result[write - 1] != '\n'; - if (implicit) { - result[write++] = '\n'; - } - result[write] = '\0'; - *out_len = write; - if (implicit_newline != NULL) { - *implicit_newline = implicit; - } -} - char * -_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, - int add_final_newline, Py_ssize_t *out_len, - int *implicit_newline) +_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, + Py_ssize_t *out_len) { - if (len > PY_SSIZE_T_MAX - 2) { + if (len < 0 || len == PY_SSIZE_T_MAX) { PyErr_NoMemory(); return NULL; } - char *result = PyMem_Malloc((size_t)len + 2); + char *result = PyMem_Malloc((size_t)len + 1); if (result == NULL) { PyErr_NoMemory(); return NULL; } - normalize_newlines_into(result, data, len, preserve_crlf, - add_final_newline, out_len, implicit_newline); + Py_ssize_t write = 0; + for (Py_ssize_t read = 0; read < len; read++) { + char c = data[read]; + if (c == '\r') { + if (read + 1 < len && data[read + 1] == '\n') { + read++; + } + c = '\n'; + } + result[write++] = c; + } + result[write] = '\0'; + *out_len = write; return result; } @@ -158,11 +140,12 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) return 0; } } - for (; i + 6 < len; i++) { - if (memcmp(line + i, "coding", 6) != 0) { + const Py_ssize_t marker_len = sizeof("coding") - 1; + for (; i < len - marker_len; i++) { + if (memcmp(line + i, "coding", marker_len) != 0) { continue; } - const char *cursor = line + i + 6; + const char *cursor = line + i + marker_len; if (*cursor != ':' && *cursor != '=') { continue; } @@ -179,37 +162,36 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) if (cursor == start) { continue; } - char *found = _PyTok_CopyBytes(start, cursor - start); - if (found == NULL) { - return -1; - } - char normalized[13]; - int n; - for (n = 0; n < 12 && found[n] != '\0'; n++) { - normalized[n] = found[n] == '_' ? '-' : Py_TOLOWER(found[n]); + Py_ssize_t encoding_len = cursor - start; + // Include the longest recognized prefix and its NUL terminator. + char normalized[sizeof("iso-latin-1-")]; + const Py_ssize_t prefix_len = Py_MIN( + encoding_len, (Py_ssize_t)sizeof(normalized) - 1); + Py_ssize_t n; + for (n = 0; n < prefix_len; n++) { + normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]); } normalized[n] = '\0'; - const char *canonical = found; + const char *canonical = start; if (strcmp(normalized, "utf-8") == 0 || - strncmp(normalized, "utf-8-", 6) == 0) { + strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) { canonical = "utf-8"; } else if (strcmp(normalized, "latin-1") == 0 || strcmp(normalized, "iso-8859-1") == 0 || strcmp(normalized, "iso-latin-1") == 0 || - strncmp(normalized, "latin-1-", 8) == 0 || - strncmp(normalized, "iso-8859-1-", 11) == 0 || - strncmp(normalized, "iso-latin-1-", 12) == 0) { + strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 || + strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 || + strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) { canonical = "iso-8859-1"; } - if (canonical != found) { - PyMem_Free(found); - found = _PyTok_CopyBytes(canonical, strlen(canonical)); - if (found == NULL) { - return -1; - } + if (canonical != start) { + encoding_len = strlen(canonical); + } + *encoding = _PyTok_CopyBytes(canonical, encoding_len); + if (*encoding == NULL) { + return -1; } - *encoding = found; *scan_next = 0; return 0; } @@ -221,17 +203,15 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, const _PyTok_Chunk *second, int final, Py_ssize_t *bom_len) { - int bom = first->len >= 3 && - (unsigned char)first->data[0] == 0xEF && - (unsigned char)first->data[1] == 0xBB && - (unsigned char)first->data[2] == 0xBF; - *bom_len = bom ? 3 : 0; + int bom = first->len >= UTF8_BOM_LENGTH && + memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0; + *bom_len = bom ? UTF8_BOM_LENGTH : 0; char *cookie = NULL; int scan_next = 0; int cookie_line = 1; - const char *first_data = first->data + (bom ? 3 : 0); - Py_ssize_t first_len = first->len - (bom ? 3 : 0); + const char *first_data = first->data + *bom_len; + Py_ssize_t first_len = first->len - *bom_len; if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) { return _PYTOK_ENCODING_ERROR; } @@ -245,19 +225,16 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, return _PYTOK_ENCODING_NEED_SECOND_LINE; } - if (bom) { - if (_PyTok_SetEncoding(tok, "utf-8") < 0) { - PyMem_Free(cookie); + if (cookie == NULL) { + if (bom && _PyTok_SetEncoding(tok, "utf-8") < 0) { return _PYTOK_ENCODING_ERROR; } - } - if (cookie == NULL) { return _PYTOK_ENCODING_DONE; } if (bom && strcmp(cookie, "utf-8") != 0) { const _PyTok_Chunk *line = cookie_line == 2 ? second : first; - const char *line_data = line->data + (cookie_line == 1 ? 3 : 0); - Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0); + const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0); + Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0); int end_col = (int)Py_MIN(line_len, INT_MAX); if (end_col > 0 && (line_data[end_col - 1] == '\n' || line_data[end_col - 1] == '\r')) { @@ -269,11 +246,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } - if (!bom && _PyTok_SetEncoding(tok, cookie) < 0) { - PyMem_Free(cookie); - return _PYTOK_ENCODING_ERROR; - } - PyMem_Free(cookie); + PyMem_Free(tok->encoding); + tok->encoding = cookie; return _PYTOK_ENCODING_DONE; } @@ -329,37 +303,46 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, data[pos + raw_line_len - 1] == '\r'; int add_newline = add_final_newline && pos + raw_line_len == len && !terminated; - int normalize = add_newline || - (!preserve_crlf && - memchr(data + pos, '\r', raw_line_len) != NULL); - const char *line = data + pos; Py_ssize_t line_len = raw_line_len; - int implicit = 0; - if (normalize) { - if (line_len > PY_SSIZE_T_MAX - 2) { + // raw_line_length stops at the first CR or LF, so any CR is in + // the line terminator. + Py_ssize_t terminator_len = 0; + if (!preserve_crlf) { + if (line[line_len - 1] == '\r') { + terminator_len = 1; + } + else if (line_len > 1 && line[line_len - 2] == '\r') { + terminator_len = 2; + } + } + if (terminator_len || add_newline) { + Py_ssize_t payload_len = line_len - terminator_len; + if (payload_len > PY_SSIZE_T_MAX - 2) { PyErr_NoMemory(); tok->done = E_NOMEM; goto error; } - // Reserve space for an optional final '\n' and the NUL terminator. - Py_ssize_t needed = line_len + 2; - if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, 256) < 0) { + Py_ssize_t needed = payload_len + 2; + if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, + NORMALIZED_LINE_INITIAL_CAPACITY) < 0) { tok->done = E_NOMEM; goto error; } - normalize_newlines_into(normalized, line, line_len, - preserve_crlf, add_newline, - &line_len, &implicit); + memcpy(normalized, line, payload_len); + normalized[payload_len] = '\n'; + line_len = payload_len + 1; + normalized[line_len] = '\0'; line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( - &tok->source, line, line_len, implicit); + &tok->source, line, line_len); if (appended < 0) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; goto error; } + tok->reader->prepared_final_newline_is_implicit = add_newline; pos += raw_line_len; } PyMem_Free(normalized); @@ -472,11 +455,9 @@ _PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final) return 0; } int strip_bom = reader->kind == _PYTOK_READER_READLINE && - chunk->len >= 2 && - (((unsigned char)chunk->data[0] == 0xFF && - (unsigned char)chunk->data[1] == 0xFE) || - ((unsigned char)chunk->data[0] == 0xFE && - (unsigned char)chunk->data[1] == 0xFF)); + chunk->len >= UTF16_BOM_LENGTH && + (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 || + memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0); PyObject *input; if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT && PyBytes_Check(chunk->owner) && diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c003deba18dd8f2..36f910e3199756d 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_fileutils.h" #include "pycore_pystate.h" +#include "pycore_runtime.h" #include "errcode.h" #include "helpers.h" @@ -8,6 +9,8 @@ #include "reader_internal.h" #include "../lexer/state.h" +#define READER_BUFFER_GROWTH_FACTOR 2 + #ifdef HAVE_UNISTD_H # include #endif @@ -37,6 +40,24 @@ _PyTok_ReaderFree(struct tok_state *tok) tok->reader = NULL; } +int +_PyTokenizer_Traverse(struct tok_state *tok, visitproc visit, void *arg) +{ + Py_VISIT(tok->filename); + Py_VISIT(tok->module); + _PyTok_Reader *reader = tok->reader; + Py_VISIT(reader->readline); + Py_VISIT(reader->decoder); + for (int i = 0; + i < (int)Py_ARRAY_LENGTH(reader->prefetched_lines); i++) { + _PyTok_Chunk *chunk = &reader->prefetched_lines[i]; + if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT) { + Py_VISIT(chunk->owner); + } + } + return 0; +} + int _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, Py_ssize_t initial_capacity) @@ -48,11 +69,11 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, // Grow geometrically to avoid reallocating for every longer line. Py_ssize_t cap = *capacity > 0 ? *capacity : initial_capacity; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / READER_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= READER_BUFFER_GROWTH_FACTOR; } char *resized = PyMem_Realloc(*buffer, cap); if (resized == NULL) { @@ -67,10 +88,14 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { + if (len == 0) { + return 0; + } if (reader->decoded_pos > 0) { Py_ssize_t remaining = reader->decoded_len - reader->decoded_pos; memmove(reader->decoded, reader->decoded + reader->decoded_pos, (size_t)remaining); + reader->decoded_scan -= reader->decoded_pos; reader->decoded_pos = 0; reader->decoded_len = remaining; } @@ -104,13 +129,17 @@ static int pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) { assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len); + assert(reader->decoded_scan >= reader->decoded_pos && + reader->decoded_scan <= reader->decoded_len); if (reader->decoded_pos == reader->decoded_len) { return 0; } char *start = reader->decoded + reader->decoded_pos; - char *newline = memchr(start, '\n', - reader->decoded_len - reader->decoded_pos); + // Previously scanned bytes cannot contain a newline. + char *newline = memchr(reader->decoded + reader->decoded_scan, '\n', + reader->decoded_len - reader->decoded_scan); if (newline == NULL) { + reader->decoded_scan = reader->decoded_len; return 0; } Py_ssize_t len = newline - start + 1; @@ -118,10 +147,12 @@ pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) chunk->len = len; chunk->ownership = _PYTOK_CHUNK_BORROWED; reader->decoded_pos += len; + reader->decoded_scan = reader->decoded_pos; chunk->implicit_newline = reader->decoded_pos == reader->decoded_len && reader->decoded_tail_is_implicit; if (reader->decoded_pos == reader->decoded_len) { reader->decoded_pos = reader->decoded_len = 0; + reader->decoded_scan = 0; reader->decoded_tail_is_implicit = 0; } return 1; @@ -142,7 +173,6 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } - int lineno = tok->lineno + 1; const char *start = _PyLexer_BufferPointer(tok, tok->inp); const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -151,8 +181,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) chunk->data = (char *)start; chunk->len = tok->source.bytes + end - start; chunk->ownership = _PYTOK_CHUNK_BORROWED; - chunk->implicit_newline = _PyTok_SourceLineIsImplicit( - &tok->source, lineno); + chunk->implicit_newline = end == tok->source.len && + tok->reader->prepared_final_newline_is_implicit; return _PYTOK_READ_LINE; } @@ -162,9 +192,12 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_Reader *reader = tok->reader; Py_ssize_t len = 0; for (;;) { - if (len > PY_SSIZE_T_MAX - BUFSIZ || - _PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, - len + BUFSIZ, BUFSIZ) < 0) { + if (len > PY_SSIZE_T_MAX - BUFSIZ) { + PyErr_NoMemory(); + return _PYTOK_READ_ERROR; + } + if (_PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, + len + BUFSIZ, BUFSIZ) < 0) { return _PYTOK_READ_ERROR; } int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX); @@ -182,7 +215,7 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) break; } } - int implicit = len == 0 || reader->file_buffer[len - 1] != '\n'; + int implicit = reader->file_buffer[len - 1] != '\n'; chunk->data = reader->file_buffer; chunk->len = len; chunk->implicit_newline = implicit; @@ -281,8 +314,7 @@ finalize_decoding(struct tok_state *tok) return -1; } } - if (reader->decoded_pos < reader->decoded_len && - append_implicit_newline(reader) < 0) { + if (append_implicit_newline(reader) < 0) { tok->done = E_NOMEM; return -1; } @@ -325,7 +357,8 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } int decoded = _PyTok_DecodeChunk(tok, &input, 0); - if (decoded == 0 && chunk_is_line(&input)) { + if (decoded == 0 && reader->decoded_pos == reader->decoded_len && + chunk_is_line(&input)) { *chunk = input; return _PYTOK_READ_LINE; } @@ -466,9 +499,6 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_ERROR; } } - if (pop_decoded_line(reader, chunk)) { - return _PYTOK_READ_LINE; - } } } @@ -503,12 +533,14 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_ChunkClear(&decoded); return _PYTOK_READ_ERROR; } + if (memchr(decoded.data, '\r', decoded.len) == NULL) { + *chunk = decoded; + return _PYTOK_READ_LINE; + } chunk->data = _PyTok_NormalizeNewlines( - decoded.data, decoded.len, 0, 0, - &chunk->len, NULL); + decoded.data, decoded.len, &chunk->len); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { - PyErr_NoMemory(); tok->done = E_NOMEM; return _PYTOK_READ_ERROR; } @@ -618,8 +650,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) reset_streaming_buffer(tok); } _PyTok_Off source_start = _PyTok_SourceAppendLine( - &tok->source, chunk.data, chunk.len, - chunk.implicit_newline); + &tok->source, chunk.data, chunk.len); if (source_start < 0) { _PyTok_ChunkClear(&chunk); tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) @@ -780,7 +811,16 @@ static FILE * fdopen_borrow(int fd) { int copy = _Py_dup(fd); - return copy < 0 ? NULL : fdopen(copy, "r"); + if (copy < 0) { + return NULL; + } + FILE *fp = fdopen(copy, "r"); + if (fp == NULL) { + int saved_errno = errno; + close(copy); + errno = saved_errno; + } + return fp; } #endif @@ -796,21 +836,14 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) fclose(fp); return NULL; } - tok->filename = filename != NULL - ? Py_NewRef(filename) : PyUnicode_FromString(""); - if (tok->filename == NULL) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; - } - if (initialize_file(tok) < 0) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; + _Py_DECLARE_STR(anon_string, ""); + tok->filename = Py_NewRef(filename != NULL ? filename : &_Py_STR(anon_string)); + char *encoding = NULL; + if (initialize_file(tok) == 0) { + encoding = tok->encoding; + tok->encoding = NULL; } fclose(fp); - char *encoding = tok->encoding == NULL - ? NULL : _PyTok_CopyBytes(tok->encoding, strlen(tok->encoding)); _PyTokenizer_Free(tok); return encoding; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 390b6ed9d8581a5..f74063de8d653a4 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -3,6 +3,8 @@ #include "Python.h" +#define _PYTOK_ENCODING_COOKIE_MAX_LINES 2 + typedef enum { _PYTOK_READER_PREPARED, _PYTOK_READER_FILE, @@ -46,13 +48,15 @@ typedef struct _PyTok_Reader { char *file_buffer; Py_ssize_t file_buffer_cap; - _PyTok_Chunk prefetched_lines[2]; + _PyTok_Chunk prefetched_lines[_PYTOK_ENCODING_COOKIE_MAX_LINES]; char *decoded; Py_ssize_t decoded_pos; + Py_ssize_t decoded_scan; // First byte not yet scanned for a newline. Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; + unsigned char prepared_final_newline_is_implicit; unsigned char decoded_tail_is_implicit; unsigned char file_initialized; unsigned char file_eof; @@ -67,8 +71,7 @@ int _PyTok_ReserveBuffer(char **, Py_ssize_t *, Py_ssize_t, Py_ssize_t); char *_PyTok_CopyBytes(const char *, Py_ssize_t); int _PyTok_DecodeOnce( struct tok_state *, _PyTok_Chunk *, const char *, const char *); -char *_PyTok_NormalizeNewlines( - const char *, Py_ssize_t, int, int, Py_ssize_t *, int *); +char *_PyTok_NormalizeNewlines(const char *, Py_ssize_t, Py_ssize_t *); void _PyTok_ChunkClear(_PyTok_Chunk *); int _PyTok_SetEncoding(struct tok_state *, const char *); _PyTok_EncodingResult _PyTok_DetectEncoding( diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index d69eab93923e81c..ebb7b05507c56ec 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -1,7 +1,10 @@ #include "Python.h" +#include "pycore_pymem.h" // PYMEM_DEADBYTE #include "source.h" +#define SOURCE_BUFFER_GROWTH_FACTOR 2 + void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -12,7 +15,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -25,10 +27,6 @@ _PyTok_SourceDiscard(_PyTok_SourceText *source) if (source->bytes != NULL) { source->bytes[0] = '\0'; } - if (source->implicit_lines != NULL) { - Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0); - memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap)); - } source->nlines = 0; } @@ -40,11 +38,11 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } Py_ssize_t cap = source->cap > 0 ? source->cap : BUFSIZ; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / SOURCE_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= SOURCE_BUFFER_GROWTH_FACTOR; } char *bytes; #ifdef Py_DEBUG @@ -62,7 +60,7 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } #ifdef Py_DEBUG if (source->bytes != NULL) { - memset(source->bytes, 0xDD, source->cap); + memset(source->bytes, PYMEM_DEADBYTE, source->cap); PyMem_Free(source->bytes); } #endif @@ -71,35 +69,9 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_implicit_lines(_PyTok_SourceText *source, int nlines) -{ - Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0); - if (needed <= source->implicit_cap) { - return 0; - } - Py_ssize_t cap = source->implicit_cap > 0 ? source->implicit_cap : 16; - while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { - cap = needed; - break; - } - cap *= 2; - } - unsigned char *lines = PyMem_Realloc(source->implicit_lines, cap); - if (lines == NULL) { - PyErr_NoMemory(); - return -1; - } - memset(lines + source->implicit_cap, 0, cap - source->implicit_cap); - source->implicit_lines = lines; - source->implicit_cap = cap; - return 0; -} - static int validate_line(const _PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { if (len <= 0 || bytes == NULL || (source->nlines > 0 && @@ -108,8 +80,7 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, return -1; } const char *newline = memchr(bytes, '\n', len); - if ((newline != NULL && newline != bytes + len - 1) || - (implicit_newline && newline == NULL)) { + if (newline != NULL && newline != bytes + len - 1) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); return -1; } @@ -123,9 +94,9 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, _PyTok_Off _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { - if (validate_line(source, bytes, len, implicit_newline) < 0) { + if (validate_line(source, bytes, len) < 0) { return -1; } if (source->len > PY_SSIZE_T_MAX - len - 1 || @@ -133,9 +104,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, PyErr_NoMemory(); return -1; } - int nlines = source->nlines + 1; - if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || - reserve_bytes(source, source->len + len + 1) < 0) { + if (reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -143,11 +112,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (implicit_newline) { - source->implicit_lines[(nlines - 1) / 8] |= - (unsigned char)(1U << ((nlines - 1) & 7)); - } - source->nlines = nlines; + source->nlines++; return source->base_offset + start; } @@ -170,14 +135,3 @@ _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, *len = (newline != NULL ? newline : end) - line; return line; } - -int -_PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) -{ - if (lineno < 1 || lineno > source->nlines || - (lineno - 1) / 8 >= source->implicit_cap) { - return 0; - } - return (source->implicit_lines[(lineno - 1) / 8] >> - ((lineno - 1) & 7)) & 1; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 9576419aade1d46..66980a41fb7c481 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -10,9 +10,7 @@ typedef struct { _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - unsigned char *implicit_lines; int nlines; - Py_ssize_t implicit_cap; } _PyTok_SourceText; static inline const char * @@ -29,19 +27,14 @@ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the - final line. implicit_newline means that the final newline was synthesized. - The input must not point into source storage. */ + final line. The input must not point into source storage. */ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( - _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, - int implicit_newline); + _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); /* Return borrowed bytes excluding '\n', writing the byte length to *len. Line numbers are 1-based and clamp to the first or final line; a trailing '\n' adds an empty final line. The view need not be NUL-terminated. This does not set an exception. Append, discard, and clear invalidate the view. */ PyAPI_FUNC(const char *) _PyTok_SourceLineView( const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); -/* Return false for invalid line numbers and the virtual EOF line. */ -PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( - const _PyTok_SourceText *, int); #endif diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index 82a84830d7cbfe7..2ef87add3646c3c 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -58,6 +58,7 @@ typedef struct { Errors are returned as ERRORTOKEN, with or without a Python exception. */ void _PyTokenizer_Get(struct tok_state *, struct token *); void _PyTokenizer_Free(struct tok_state *); +int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *); void _PyTokenizer_raise_init_error(PyObject *filename); void _PyToken_Init(struct token *); static inline void diff --git a/Parser/tokenizer/types.h b/Parser/tokenizer/types.h index c12b6de449c7008..b488cb01618d35f 100644 --- a/Parser/tokenizer/types.h +++ b/Parser/tokenizer/types.h @@ -18,12 +18,6 @@ typedef struct { int byte_col; } _PyTok_Loc; -static inline _PyTok_Span -_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) -{ - return (_PyTok_Span){start, end}; -} - static inline int _PyTok_SpanIsValid(_PyTok_Span span) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 3802709732e29da..e91141bd1b9e610 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -1,7 +1,6 @@ #include "Python.h" #include "errcode.h" #include "internal/pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION -#include "internal/pycore_tuple.h" // _PyTuple_FromPair #include "../Parser/tokenizer/tokenizer.h" #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() @@ -37,7 +36,6 @@ typedef struct /* Needed to cache line for performance */ PyObject *last_line; Py_ssize_t last_lineno; - Py_ssize_t last_end_lineno; Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; @@ -57,21 +55,18 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, int extra_tokens, const char *encoding) /*[clinic end generated code: output=7501a1211683ce16 input=f7dddf8a613ae8bd]*/ { - tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); - if (self == NULL) { + struct tok_state *tok = _PyTokenizer_FromReadline(readline, encoding); + if (tok == NULL) { return NULL; } - PyObject *filename = PyUnicode_FromString(""); - if (filename == NULL) { - return NULL; - } - self->tok = _PyTokenizer_FromReadline(readline, encoding); - if (self->tok == NULL) { - Py_DECREF(filename); + _Py_DECLARE_STR(anon_string, ""); + _PyTokenizer_SetContext(tok, &_Py_STR(anon_string), NULL); + tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); + if (self == NULL) { + _PyTokenizer_Free(tok); return NULL; } - _PyTokenizer_SetContext(self->tok, filename, NULL); - Py_DECREF(filename); + self->tok = tok; _PyTokenizer_SetOptions(self->tok, extra_tokens, 0); self->extra_tokens = extra_tokens; self->done = 0; @@ -79,18 +74,15 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->last_line = NULL; self->byte_col_offset_diff = 0; self->last_lineno = 0; - self->last_end_lineno = 0; return (PyObject *)self; } -static int +static void _tokenizer_error(tokenizeriterobject *it) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - if (PyErr_Occurred()) { - return -1; - } + assert(!PyErr_Occurred()); const char *msg = NULL; PyObject* errtype = PyExc_SyntaxError; @@ -105,19 +97,17 @@ _tokenizer_error(tokenizeriterobject *it) PyErr_SyntaxLocationObject( info.filename, info.location.lineno, (int)Py_MAX(0, info.input_span.end - info.input_span.start)); - return -1; + return; case E_DEDENT: msg = "unindent does not match any outer indentation level"; errtype = PyExc_IndentationError; break; case E_INTR: - if (!PyErr_Occurred()) { - PyErr_SetNone(PyExc_KeyboardInterrupt); - } - return -1; + PyErr_SetNone(PyExc_KeyboardInterrupt); + return; case E_NOMEM: PyErr_NoMemory(); - return -1; + return; case E_TABSPACE: errtype = PyExc_TabError; msg = "inconsistent use of tabs and spaces in indentation"; @@ -134,11 +124,8 @@ _tokenizer_error(tokenizeriterobject *it) msg = "unknown tokenization error"; } - PyObject* errstr = NULL; PyObject* error_line = NULL; - PyObject* tmp = NULL; PyObject* value = NULL; - int result = 0; Py_ssize_t input_size; const char *input = _PyTokenizer_SpanView( @@ -148,41 +135,25 @@ _tokenizer_error(tokenizeriterobject *it) size -= 1; // Remove the newline character from the end of the line error_line = PyUnicode_DecodeUTF8(input, size, "replace"); if (!error_line) { - result = -1; goto exit; } Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size); if (offset == -1) { - result = -1; - goto exit; - } - tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno, offset, error_line, Py_None, Py_None); - if (!tmp) { - result = -1; - goto exit; - } - - errstr = PyUnicode_FromString(msg); - if (!errstr) { - result = -1; goto exit; } - - value = _PyTuple_FromPair(errstr, tmp); + value = Py_BuildValue("(s(OinOOO))", msg, info.filename, + info.location.lineno, offset, error_line, + Py_None, Py_None); if (!value) { - result = -1; goto exit; } PyErr_SetObject(errtype, value); exit: - Py_XDECREF(errstr); Py_XDECREF(error_line); - Py_XDECREF(tmp); Py_XDECREF(value); - return result; } static PyObject * @@ -190,38 +161,38 @@ _get_current_line(tokenizeriterobject *it, int current_lineno, const char *line_start, Py_ssize_t size, int *line_changed) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - PyObject *line; if (current_lineno != it->last_lineno) { // Line has changed since last token, so we fetch the new line and cache it // in the iter object. Py_XDECREF(it->last_line); - line = PyUnicode_DecodeUTF8(line_start, size, "replace"); - it->last_line = line; + it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace"); it->byte_col_offset_diff = 0; } else { - line = it->last_line; *line_changed = 0; } - return line; + return it->last_line; } -static void +static int _get_col_offsets(tokenizeriterobject *it, const struct token *token, - const char *token_start, const char *line_start, - const char *end_line_start, PyObject *line, int line_changed, + const _PyToken_View *view, PyObject *line, int line_changed, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + const char *token_start = view->text; const char *token_end = token_start == NULL - ? NULL : token_start + token->span.end - token->span.start; + ? NULL : token_start + view->length; Py_ssize_t lineno = token->start_loc.lineno; Py_ssize_t end_lineno = token->end_loc.lineno; Py_ssize_t byte_offset = -1; - if (token_start != NULL && token_start >= line_start) { - byte_offset = token_start - line_start; + if (token_start != NULL && token_start >= view->line) { + byte_offset = token_start - view->line; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + if (*col_offset < 0) { + return -1; + } it->byte_col_offset_diff = byte_offset - *col_offset; } else { @@ -229,21 +200,28 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, } } - if (token_end != NULL && token_end >= end_line_start) { - Py_ssize_t end_byte_offset = token_end - end_line_start; + if (token_end != NULL && token_end >= view->end_line) { + Py_ssize_t end_byte_offset = token_end - view->end_line; if (lineno == end_lineno) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + if (token_col_offset < 0) { + return -1; + } *end_col_offset = *col_offset + token_col_offset; it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { - *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset); + *end_col_offset = _PyPegen_byte_offset_to_character_offset_line( + line, view->end_line - view->line, token_end - view->line); + if (*end_col_offset < 0) { + return -1; + } it->byte_col_offset_diff += end_byte_offset - *end_col_offset; } } it->last_lineno = lineno; - it->last_end_lineno = end_lineno; + return 0; } static PyObject * @@ -254,6 +232,11 @@ tokenizeriter_next(PyObject *op) Py_BEGIN_CRITICAL_SECTION(it); + if (it->done) { + PyErr_SetString(PyExc_StopIteration, "EOF"); + goto unlock; + } + struct token token; _PyToken_Init(&token); @@ -266,11 +249,6 @@ tokenizeriter_next(PyObject *op) } goto exit; } - if (it->done || type == ERRORTOKEN) { - PyErr_SetString(PyExc_StopIteration, "EOF"); - it->done = 1; - goto exit; - } _PyToken_View view; _PyToken_GetView(it->tok, &token, &view); const char *token_start = view.text; @@ -286,10 +264,7 @@ tokenizeriter_next(PyObject *op) goto exit; } - int is_trailing_token = 0; - if (type == ENDMARKER || (type == DEDENT && view.at_eof)) { - is_trailing_token = 1; - } + int is_trailing_token = type == ENDMARKER || (type == DEDENT && view.at_eof); PyObject* line = NULL; int line_changed = 1; @@ -313,8 +288,11 @@ tokenizeriter_next(PyObject *op) Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, &token, token_start, view.line, view.end_line, line, - line_changed, &col_offset, &end_col_offset); + if (_get_col_offsets(it, &token, &view, line, + line_changed, &col_offset, &end_col_offset) < 0) { + Py_DECREF(str); + goto exit; + } if (it->extra_tokens) { if (is_trailing_token) { @@ -327,8 +305,8 @@ tokenizeriter_next(PyObject *op) type = OP; } else if (type == NEWLINE) { - Py_DECREF(str); if (!view.implicit_newline) { + Py_DECREF(str); assert(token_start != NULL); if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); @@ -346,7 +324,6 @@ tokenizeriter_next(PyObject *op) } if (str == NULL) { - Py_DECREF(line); goto exit; } } @@ -358,6 +335,7 @@ tokenizeriter_next(PyObject *op) it->done = 1; } +unlock:; Py_END_CRITICAL_SECTION(); return result; } @@ -367,15 +345,26 @@ tokenizeriter_dealloc(PyObject *op) { tokenizeriterobject *it = (tokenizeriterobject*)op; PyTypeObject *tp = Py_TYPE(it); + PyObject_GC_UnTrack(it); Py_XDECREF(it->last_line); _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); } +static int +tokenizeriter_traverse(PyObject *op, visitproc visit, void *arg) +{ + tokenizeriterobject *it = (tokenizeriterobject *)op; + Py_VISIT(Py_TYPE(it)); + Py_VISIT(it->last_line); + return _PyTokenizer_Traverse(it->tok, visit, arg); +} + static PyType_Slot tokenizeriter_slots[] = { {Py_tp_new, tokenizeriter_new}, {Py_tp_dealloc, tokenizeriter_dealloc}, + {Py_tp_traverse, tokenizeriter_traverse}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, tokenizeriter_next}, @@ -385,7 +374,7 @@ static PyType_Slot tokenizeriter_slots[] = { static PyType_Spec tokenizeriter_spec = { .name = "_tokenize.TokenizerIter", .basicsize = sizeof(tokenizeriterobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), .slots = tokenizeriter_slots, }; diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 6dd598beb1b39c4..f6cf2d954e4e7bb 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -31,12 +31,6 @@ def emit(self, value: str) -> None: self.writer.print("p->level--;") self.writer.print(f"return {value};") - def check_memory(self, expr: str) -> None: - self.writer.print(f"if ({expr}) {{") - with self.writer.indent(): - self.no_memory() - self.writer.print("}") - def no_memory(self) -> None: self.writer.print("p->error_indicator = 1;") self.writer.print("PyErr_NoMemory();") @@ -51,24 +45,19 @@ class _LoopBuffer: def __init__(self, writer: CWriter, returns: _CReturnEmitter): self._print = writer.print self._indent = writer.indent - self._returns = returns self.error_returns = returns.with_cleanup(self._release) def initialize(self) -> None: - self._print("void **_children = PyMem_Malloc(sizeof(void *));") - self._returns.check_memory("!_children") - self._print("Py_ssize_t _children_capacity = 1;") + self._print("void **_children = NULL;") + self._print("Py_ssize_t _children_capacity = 0;") self._print("Py_ssize_t _n = 0;") def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print("_children_capacity *= 2;") - self._print( - "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" + self._check_memory( + "_PyPegen_grow_loop_buffer(&_children, &_children_capacity) < 0" ) - self._check_memory("!_new_children") - self._print("_children = _new_children;") self._print("}") self._print(f"_children[_n++] = {value};") @@ -87,8 +76,7 @@ def finish(self, *, require_one: bool) -> str: def _check_memory(self, expr: str) -> None: self._print(f"if ({expr}) {{") with self._indent(): - self._print(self._release) - self._returns.no_memory() + self.error_returns.no_memory() self._print("}")