Oct 2, 2023 · Sep 22, 2023
diff --git a/Grammar/python.gram b/Grammar/python.gram
        result = eval_rule(p);
    } else if (p->start_rule == Py_func_type_input) {
        result = func_type_rule(p);
    } else if (p->start_rule == Py_fstring_input) {
        result = fstring_rule(p);
    }

    return result;
 interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(a, p->arena) }
 eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { _PyAST_Expression(a, p->arena) }
 func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { _PyAST_FunctionType(a, b, p->arena) }
 fstring[expr_ty]: star_expressions

 # GENERAL STATEMENTS
 # ==================

 type_param[type_param_ty] (memo):
    | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) }
    | '*' a=NAME colon=":" e=expression {
    | '*' a=NAME colon=':' e=expression {
            RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
                ? "cannot use constraints with TypeVarTuple"
                : "cannot use bound with TypeVarTuple")
        }
    | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) }
    | '**' a=NAME colon=":" e=expression {
    | '**' a=NAME colon=':' e=expression {
            RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
                ? "cannot use constraints with ParamSpec"
                : "cannot use bound with ParamSpec")
        }
    | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) }

 type_param_bound[expr_ty]:":" e=expression { e }
 type_param_bound[expr_ty]:':' e=expression { e }

 # EXPRESSIONS
 # -----------
    | fstring_replacement_field
    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
 fstring_replacement_field[expr_ty]:
    | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
    | '{' a=(yield_expr | star_expressions) debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
        _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
    | invalid_replacement_field
 fstring_conversion[ResultTokenWithMetadata*]:
diff --git a/Include/compile.h b/Include/compile.h
 #define Py_eval_input 258
 #define Py_func_type_input 345

 /* This doesn't need to match anything */
 #define Py_fstring_input 800

 #ifndef Py_LIMITED_API
 #  define Py_CPYTHON_COMPILE_H
 #  include "cpython/compile.h"
diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
        )
        self.assertEqual(repr(rules["term"]), expected_repr)

    def test_repeated_rules(self) -> None:
        grammar_source = """
        start: the_rule NEWLINE
        the_rule: 'b' NEWLINE
        the_rule: 'a' NEWLINE
        """
        with self.assertRaisesRegex(GrammarError, "Repeated rule 'the_rule'"):
            parse_string(grammar_source, GrammarParser)

    def test_long_rule_str(self) -> None:
        grammar_source = """
        start: zero | one | one zero | one one | one zero zero | one zero one | one one zero | one one one
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-20-13-18-08.gh-issue-109596.RG0K2G.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-20-13-18-08.gh-issue-109596.RG0K2G.rst
 Fix some tokens in the grammar that were incorrectly marked as soft
 keywords. Also fix some repeated rule names and ensure that repeated rules
 are not allowed. Patch by Pablo Galindo
Original file line number	Diff line number	Diff line change
Expand Up		@@ -19,8 +19,6 @@ _PyPegen_parse(Parser *p)
		result = eval_rule(p);
		} else if (p->start_rule == Py_func_type_input) {
		result = func_type_rule(p);
		} else if (p->start_rule == Py_fstring_input) {
		result = fstring_rule(p);
		}

		return result;
Expand DownExpand Up		@@ -89,7 +87,6 @@ file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) }
		interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(a, p->arena) }
		eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { _PyAST_Expression(a, p->arena) }
		func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { _PyAST_FunctionType(a, b, p->arena) }
		fstring[expr_ty]: star_expressions

		# GENERAL STATEMENTS
		# ==================
Expand DownExpand Up		@@ -647,20 +644,20 @@ type_param_seq[asdl_type_param_seq]: a[asdl_type_param_seq]=','.type_param+ ['

		type_param[type_param_ty] (memo):
		\| a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) }
		\| '*' a=NAME colon=":" e=expression {
		\| '*' a=NAME colon=':' e=expression {
		RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
		? "cannot use constraints with TypeVarTuple"
		: "cannot use bound with TypeVarTuple")
		}
		\| '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) }
		\| '**' a=NAME colon=":" e=expression {
		\| '**' a=NAME colon=':' e=expression {
		RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
		? "cannot use constraints with ParamSpec"
		: "cannot use bound with ParamSpec")
		}
		\| '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) }

		type_param_bound[expr_ty]:":" e=expression { e }
		type_param_bound[expr_ty]:':' e=expression { e }

		# EXPRESSIONS
		# -----------
Expand DownExpand Up		@@ -915,7 +912,7 @@ fstring_middle[expr_ty]:
		\| fstring_replacement_field
		\| t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
		fstring_replacement_field[expr_ty]:
		\| '{' a=(yield_expr \| star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
		\| '{' a=(yield_expr \| star_expressions) debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
		_PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
		\| invalid_replacement_field
		fstring_conversion[ResultTokenWithMetadata*]:
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,9 +10,6 @@ extern "C" {
		#define Py_eval_input 258
		#define Py_func_type_input 345

		/* This doesn't need to match anything */
		#define Py_fstring_input 800

		#ifndef Py_LIMITED_API
		# define Py_CPYTHON_COMPILE_H
		# include "cpython/compile.h"
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -42,6 +42,15 @@ def test_parse_grammar(self) -> None:
		)
		self.assertEqual(repr(rules["term"]), expected_repr)

		def test_repeated_rules(self) -> None:
		grammar_source = """
		start: the_rule NEWLINE
		the_rule: 'b' NEWLINE
		the_rule: 'a' NEWLINE
		"""
		with self.assertRaisesRegex(GrammarError, "Repeated rule 'the_rule'"):
		parse_string(grammar_source, GrammarParser)

		def test_long_rule_str(self) -> None:
		grammar_source = """
		start: zero \| one \| one zero \| one one \| one zero zero \| one zero one \| one one zero \| one one one
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		Fix some tokens in the grammar that were incorrectly marked as soft
		keywords. Also fix some repeated rule names and ensure that repeated rules
		are not allowed. Patch by Pablo Galindo