Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit95e519e

Browse files
committed
RemoveNtIdent andNtLifetime.
The extra span is now recorded in the new `TokenKind::NtIdent` and`TokenKind::NtLifetime`. These both consist of a single token, and sothere's no operator precedence problems with inserting them directlyinto the token stream.The other way to do this would be to wrap the ident/lifetime in invisibledelimiters, but there's a lot of code that assumes an interpolatedident/lifetime fits in a single token, and changing all that code to work withinvisible delimiters would have been a pain. (Maybe it could be done in afollow-up.)This change might not seem like much of a win, but it's a first step toward themuch bigger and long-desired removal of `Nonterminal` and`TokenKind::Interpolated`. That change is big and complex enough that it'sworth doing this piece separately. (Indeed, this commit is based on part of alate commit in#114647, a prior attempt at that big and complex change.)
1 parent9a63a42 commit95e519e

File tree

11 files changed

+131
-104
lines changed

11 files changed

+131
-104
lines changed

‎compiler/rustc_ast/src/ast_traits.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,6 @@ impl HasTokens for Nonterminal {
240240
Nonterminal::NtPath(path) => path.tokens(),
241241
Nonterminal::NtVis(vis) => vis.tokens(),
242242
Nonterminal::NtBlock(block) => block.tokens(),
243-
Nonterminal::NtIdent(..) |Nonterminal::NtLifetime(..) =>None,
244243
}
245244
}
246245
fntokens_mut(&mutself) ->Option<&mutOption<LazyAttrTokenStream>>{
@@ -254,7 +253,6 @@ impl HasTokens for Nonterminal {
254253
Nonterminal::NtPath(path) => path.tokens_mut(),
255254
Nonterminal::NtVis(vis) => vis.tokens_mut(),
256255
Nonterminal::NtBlock(block) => block.tokens_mut(),
257-
Nonterminal::NtIdent(..) |Nonterminal::NtLifetime(..) =>None,
258256
}
259257
}
260258
}

‎compiler/rustc_ast/src/mut_visit.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,12 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
781781
*span = ident.span;
782782
return;// Avoid visiting the span for the second time.
783783
}
784+
token::NtIdent(ident, _is_raw) =>{
785+
vis.visit_ident(ident);
786+
}
787+
token::NtLifetime(ident) =>{
788+
vis.visit_ident(ident);
789+
}
784790
token::Interpolated(nt) =>{
785791
let nt =Lrc::make_mut(nt);
786792
visit_nonterminal(nt, vis);
@@ -832,8 +838,6 @@ fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
832838
token::NtPat(pat) => vis.visit_pat(pat),
833839
token::NtExpr(expr) => vis.visit_expr(expr),
834840
token::NtTy(ty) => vis.visit_ty(ty),
835-
token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
836-
token::NtLifetime(ident) => vis.visit_ident(ident),
837841
token::NtLiteral(expr) => vis.visit_expr(expr),
838842
token::NtMeta(item) =>{
839843
letAttrItem{ path, args, tokens} = item.deref_mut();

‎compiler/rustc_ast/src/token.rs‎

Lines changed: 32 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -318,11 +318,20 @@ pub enum TokenKind {
318318
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
319319
/// treat regular and interpolated identifiers in the same way.
320320
Ident(Symbol,IdentIsRaw),
321+
/// This identifier (and its span) is the identifier passed to the
322+
/// declarative macro. The span in the surrounding `Token` is the span of
323+
/// the `ident` metavariable in the macro's RHS.
324+
NtIdent(Ident,IdentIsRaw),
325+
321326
/// Lifetime identifier token.
322327
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
323328
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
324329
/// treat regular and interpolated lifetime identifiers in the same way.
325330
Lifetime(Symbol),
331+
/// This identifier (and its span) is the lifetime passed to the
332+
/// declarative macro. The span in the surrounding `Token` is the span of
333+
/// the `lifetime` metavariable in the macro's RHS.
334+
NtLifetime(Ident),
326335

327336
/// An embedded AST node, as produced by a macro. This only exists for
328337
/// historical reasons. We'd like to get rid of it, for multiple reasons.
@@ -444,8 +453,9 @@ impl Token {
444453
/// Note that keywords are also identifiers, so they should use this
445454
/// if they keep spans or perform edition checks.
446455
pubfnuninterpolated_span(&self) ->Span{
447-
match&self.kind{
448-
Interpolated(nt) => nt.use_span(),
456+
matchself.kind{
457+
NtIdent(ident, _) |NtLifetime(ident) => ident.span,
458+
Interpolated(ref nt) => nt.use_span(),
449459
_ =>self.span,
450460
}
451461
}
@@ -463,7 +473,7 @@ impl Token {
463473
}
464474

465475
OpenDelim(..) |CloseDelim(..) |Literal(..) |DocComment(..) |Ident(..)
466-
|Lifetime(..) |Interpolated(..) |Eof =>false,
476+
|NtIdent(..) |Lifetime(..) |NtLifetime(..) |Interpolated(..) |Eof =>false,
467477
}
468478
}
469479

@@ -613,14 +623,9 @@ impl Token {
613623
/// into the regular identifier or lifetime token it refers to,
614624
/// otherwise returns the original token.
615625
pubfnuninterpolate(&self) ->Cow<'_,Token>{
616-
match&self.kind{
617-
Interpolated(nt) =>match&**nt{
618-
NtIdent(ident, is_raw) =>{
619-
Cow::Owned(Token::new(Ident(ident.name,*is_raw), ident.span))
620-
}
621-
NtLifetime(ident) =>Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
622-
_ =>Cow::Borrowed(self),
623-
},
626+
matchself.kind{
627+
NtIdent(ident, is_raw) =>Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)),
628+
NtLifetime(ident) =>Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
624629
_ =>Cow::Borrowed(self),
625630
}
626631
}
@@ -629,12 +634,9 @@ impl Token {
629634
#[inline]
630635
pubfnident(&self) ->Option<(Ident,IdentIsRaw)>{
631636
// We avoid using `Token::uninterpolate` here because it's slow.
632-
match&self.kind{
633-
&Ident(name, is_raw) =>Some((Ident::new(name,self.span), is_raw)),
634-
Interpolated(nt) =>match&**nt{
635-
NtIdent(ident, is_raw) =>Some((*ident,*is_raw)),
636-
_ =>None,
637-
},
637+
matchself.kind{
638+
Ident(name, is_raw) =>Some((Ident::new(name,self.span), is_raw)),
639+
NtIdent(ident, is_raw) =>Some((ident, is_raw)),
638640
_ =>None,
639641
}
640642
}
@@ -643,12 +645,9 @@ impl Token {
643645
#[inline]
644646
pubfnlifetime(&self) ->Option<Ident>{
645647
// We avoid using `Token::uninterpolate` here because it's slow.
646-
match&self.kind{
647-
&Lifetime(name) =>Some(Ident::new(name,self.span)),
648-
Interpolated(nt) =>match&**nt{
649-
NtLifetime(ident) =>Some(*ident),
650-
_ =>None,
651-
},
648+
matchself.kind{
649+
Lifetime(name) =>Some(Ident::new(name,self.span)),
650+
NtLifetime(ident) =>Some(ident),
652651
_ =>None,
653652
}
654653
}
@@ -837,8 +836,10 @@ impl Token {
837836

838837
Le |EqEq |Ne |Ge |AndAnd |OrOr |Tilde |BinOpEq(..) |At |DotDotDot
839838
|DotDotEq |Comma |Semi |PathSep |RArrow |LArrow |FatArrow |Pound |Dollar
840-
|Question |OpenDelim(..) |CloseDelim(..) |Literal(..) |Ident(..)
841-
|Lifetime(..) |Interpolated(..) |DocComment(..) |Eof =>returnNone,
839+
|Question |OpenDelim(..) |CloseDelim(..) |Literal(..) |Ident(..) |NtIdent(..)
840+
|Lifetime(..) |NtLifetime(..) |Interpolated(..) |DocComment(..) |Eof =>{
841+
returnNone;
842+
}
842843
};
843844

844845
Some(Token::new(kind,self.span.to(joint.span)))
@@ -861,9 +862,6 @@ pub enum Nonterminal {
861862
NtPat(P<ast::Pat>),
862863
NtExpr(P<ast::Expr>),
863864
NtTy(P<ast::Ty>),
864-
/// The span is for the identifier argument passed to the macro.
865-
NtIdent(Ident,IdentIsRaw),
866-
NtLifetime(Ident),
867865
NtLiteral(P<ast::Expr>),
868866
/// Stuff inside brackets for attributes
869867
NtMeta(P<ast::AttrItem>),
@@ -958,7 +956,6 @@ impl Nonterminal {
958956
NtPat(pat) => pat.span,
959957
NtExpr(expr) |NtLiteral(expr) => expr.span,
960958
NtTy(ty) => ty.span,
961-
NtIdent(ident, _) |NtLifetime(ident) => ident.span,
962959
NtMeta(attr_item) => attr_item.span(),
963960
NtPath(path) => path.span,
964961
NtVis(vis) => vis.span,
@@ -974,8 +971,6 @@ impl Nonterminal {
974971
NtExpr(..) =>"expression",
975972
NtLiteral(..) =>"literal",
976973
NtTy(..) =>"type",
977-
NtIdent(..) =>"identifier",
978-
NtLifetime(..) =>"lifetime",
979974
NtMeta(..) =>"attribute",
980975
NtPath(..) =>"path",
981976
NtVis(..) =>"visibility",
@@ -984,18 +979,12 @@ impl Nonterminal {
984979
}
985980

986981
implPartialEqforNonterminal{
987-
fneq(&self,rhs:&Self) ->bool{
988-
match(self, rhs){
989-
(NtIdent(ident_lhs, is_raw_lhs),NtIdent(ident_rhs, is_raw_rhs)) =>{
990-
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
991-
}
992-
(NtLifetime(ident_lhs),NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
993-
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
994-
// correctly based on data from AST. This will prevent them from matching each other
995-
// in macros. The comparison will become possible only when each nonterminal has an
996-
// attached token stream from which it was parsed.
997-
_ =>false,
998-
}
982+
fneq(&self,_rhs:&Self) ->bool{
983+
// FIXME: Assume that all nonterminals are not equal, we can't compare them
984+
// correctly based on data from AST. This will prevent them from matching each other
985+
// in macros. The comparison will become possible only when each nonterminal has an
986+
// attached token stream from which it was parsed.
987+
false
999988
}
1000989
}
1001990

@@ -1008,12 +997,10 @@ impl fmt::Debug for Nonterminal {
1008997
NtPat(..) => f.pad("NtPat(..)"),
1009998
NtExpr(..) => f.pad("NtExpr(..)"),
1010999
NtTy(..) => f.pad("NtTy(..)"),
1011-
NtIdent(..) => f.pad("NtIdent(..)"),
10121000
NtLiteral(..) => f.pad("NtLiteral(..)"),
10131001
NtMeta(..) => f.pad("NtMeta(..)"),
10141002
NtPath(..) => f.pad("NtPath(..)"),
10151003
NtVis(..) => f.pad("NtVis(..)"),
1016-
NtLifetime(..) => f.pad("NtLifetime(..)"),
10171004
}
10181005
}
10191006
}

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -466,12 +466,6 @@ impl TokenStream {
466466

467467
pubfnfrom_nonterminal_ast(nt:&Nonterminal) ->TokenStream{
468468
match nt{
469-
Nonterminal::NtIdent(ident, is_raw) =>{
470-
TokenStream::token_alone(token::Ident(ident.name,*is_raw), ident.span)
471-
}
472-
Nonterminal::NtLifetime(ident) =>{
473-
TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
474-
}
475469
Nonterminal::NtItem(item) =>TokenStream::from_ast(item),
476470
Nonterminal::NtBlock(block) =>TokenStream::from_ast(block),
477471
Nonterminal::NtStmt(stmt)ifletStmtKind::Empty = stmt.kind =>{
@@ -489,11 +483,17 @@ impl TokenStream {
489483
}
490484

491485
fnflatten_token(token:&Token,spacing:Spacing) ->TokenTree{
492-
match&token.kind{
493-
token::Interpolated(nt)iflet token::NtIdent(ident, is_raw) =&**nt =>{
494-
TokenTree::Token(Token::new(token::Ident(ident.name,*is_raw), ident.span), spacing)
486+
match token.kind{
487+
token::NtIdent(ident, is_raw) =>{
488+
TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
495489
}
496-
token::Interpolated(nt) =>TokenTree::Delimited(
490+
token::NtLifetime(ident) =>TokenTree::Delimited(
491+
DelimSpan::from_single(token.span),
492+
DelimSpacing::new(Spacing::JointHidden, spacing),
493+
Delimiter::Invisible,
494+
TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
495+
),
496+
token::Interpolated(ref nt) =>TokenTree::Delimited(
497497
DelimSpan::from_single(token.span),
498498
DelimSpacing::new(Spacing::JointHidden, spacing),
499499
Delimiter::Invisible,
@@ -516,7 +516,10 @@ impl TokenStream {
516516
pubfnflattened(&self) ->TokenStream{
517517
fncan_skip(stream:&TokenStream) ->bool{
518518
stream.trees().all(|tree|match tree{
519-
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
519+
TokenTree::Token(token, _) => !matches!(
520+
token.kind,
521+
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
522+
),
520523
TokenTree::Delimited(.., inner) =>can_skip(inner),
521524
})
522525
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -852,8 +852,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
852852
token::NtBlock(e) =>self.block_to_string(e),
853853
token::NtStmt(e) =>self.stmt_to_string(e),
854854
token::NtPat(e) =>self.pat_to_string(e),
855-
&token::NtIdent(e, is_raw) =>IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(),
856-
token::NtLifetime(e) => e.to_string(),
857855
token::NtLiteral(e) =>self.expr_to_string(e),
858856
token::NtVis(e) =>self.vis_to_string(e),
859857
}
@@ -915,10 +913,14 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
915913
token::Literal(lit) =>literal_to_string(lit).into(),
916914

917915
/* Name components */
918-
token::Ident(s, is_raw) =>{
919-
IdentPrinter::new(s, is_raw.into(), convert_dollar_crate).to_string().into()
916+
token::Ident(name, is_raw) =>{
917+
IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into()
920918
}
921-
token::Lifetime(s) => s.to_string().into(),
919+
token::NtIdent(ident, is_raw) =>{
920+
IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into()
921+
}
922+
token::Lifetime(name) => name.to_string().into(),
923+
token::NtLifetime(ident) => ident.name.to_string().into(),
922924

923925
/* Other */
924926
token::DocComment(comment_kind, attr_style, data) =>{

‎compiler/rustc_expand/src/config.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -201,10 +201,17 @@ impl<'a> StripUnconfigured<'a> {
201201
inner =self.configure_tokens(&inner);
202202
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
203203
}
204-
AttrTokenTree::Token(ref token, _)
205-
ifletTokenKind::Interpolated(nt) =&token.kind =>
206-
{
207-
panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt);
204+
AttrTokenTree::Token(
205+
Token{
206+
kind:
207+
TokenKind::NtIdent(..)
208+
|TokenKind::NtLifetime(..)
209+
|TokenKind::Interpolated(..),
210+
..
211+
},
212+
_,
213+
) =>{
214+
panic!("Nonterminal should have been flattened: {:?}", tree);
208215
}
209216
AttrTokenTree::Token(token, spacing) =>{
210217
Some(AttrTokenTree::Token(token, spacing)).into_iter()

‎compiler/rustc_expand/src/mbe/transcribe.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,16 @@ pub(super) fn transcribe<'a>(
261261
// without wrapping them into groups.
262262
maybe_use_metavar_location(cx,&stack, sp, tt,&mut marker)
263263
}
264+
MatchedSingle(ParseNtResult::Ident(ident, is_raw)) =>{
265+
marker.visit_span(&mut sp);
266+
let kind = token::NtIdent(*ident,*is_raw);
267+
TokenTree::token_alone(kind, sp)
268+
}
269+
MatchedSingle(ParseNtResult::Lifetime(ident)) =>{
270+
marker.visit_span(&mut sp);
271+
let kind = token::NtLifetime(*ident);
272+
TokenTree::token_alone(kind, sp)
273+
}
264274
MatchedSingle(ParseNtResult::Nt(nt)) =>{
265275
// Other variables are emitted into the output stream as groups with
266276
// `Delimiter::Invisible` to maintain parsing priorities.

‎compiler/rustc_expand/src/proc_macro_server.rs‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -220,13 +220,28 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
220220
Ident(sym, is_raw) =>{
221221
trees.push(TokenTree::Ident(Ident{ sym,is_raw: is_raw.into(), span}))
222222
}
223+
NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident{
224+
sym: ident.name,
225+
is_raw: is_raw.into(),
226+
span: ident.span,
227+
})),
228+
223229
Lifetime(name) =>{
224230
let ident = symbol::Ident::new(name, span).without_first_quote();
225231
trees.extend([
226232
TokenTree::Punct(Punct{ch:b'\'',joint:true, span}),
227233
TokenTree::Ident(Ident{sym: ident.name,is_raw:false, span}),
228234
]);
229235
}
236+
NtLifetime(ident) =>{
237+
let stream =TokenStream::token_alone(token::Lifetime(ident.name), ident.span);
238+
trees.push(TokenTree::Group(Group{
239+
delimiter: pm::Delimiter::None,
240+
stream:Some(stream),
241+
span:DelimSpan::from_single(span),
242+
}))
243+
}
244+
230245
Literal(token::Lit{ kind, symbol, suffix}) =>{
231246
trees.push(TokenTree::Literal(self::Literal{
232247
kind:FromInternal::from_internal(kind),
@@ -259,14 +274,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
259274
}));
260275
}
261276

262-
Interpolated(ref nt)ifletNtIdent(ident, is_raw) =&**nt =>{
263-
trees.push(TokenTree::Ident(Ident{
264-
sym: ident.name,
265-
is_raw:matches!(is_raw,IdentIsRaw::Yes),
266-
span: ident.span,
267-
}))
268-
}
269-
270277
Interpolated(nt) =>{
271278
let stream =TokenStream::from_nonterminal_ast(&nt);
272279
// A hack used to pass AST fragments to attribute and derive

‎compiler/rustc_parse/src/parser/expr.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -724,7 +724,9 @@ impl<'a> Parser<'a> {
724724
/// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
725725
fninterpolated_or_expr_span(&self,expr:&Expr) ->Span{
726726
matchself.prev_token.kind{
727-
TokenKind::Interpolated(..) =>self.prev_token.span,
727+
TokenKind::NtIdent(..) |TokenKind::NtLifetime(..) |TokenKind::Interpolated(..) =>{
728+
self.prev_token.span
729+
}
728730
_ => expr.span,
729731
}
730732
}

‎compiler/rustc_parse/src/parser/mod.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,8 @@ pub(super) fn token_descr(token: &Token) -> String {
407407
(Some(TokenDescription::Keyword), _) =>Some("keyword"),
408408
(Some(TokenDescription::ReservedKeyword), _) =>Some("reserved keyword"),
409409
(Some(TokenDescription::DocComment), _) =>Some("doc comment"),
410+
(None,TokenKind::NtIdent(..)) =>Some("identifier"),
411+
(None,TokenKind::NtLifetime(..)) =>Some("lifetime"),
410412
(None,TokenKind::Interpolated(node)) =>Some(node.descr()),
411413
(None, _) =>None,
412414
};
@@ -1633,5 +1635,9 @@ pub enum FlatToken {
16331635
#[derive(Clone,Debug)]
16341636
pubenumParseNtResult{
16351637
Tt(TokenTree),
1638+
Ident(Ident,IdentIsRaw),
1639+
Lifetime(Ident),
1640+
1641+
/// This case will eventually be removed, along with `Token::Interpolate`.
16361642
Nt(Lrc<Nonterminal>),
16371643
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp