Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit62c068f

Browse files
committed
Auto merge of#127636 - nnethercote:fix-Parser-look_ahead, r=oli-obk
Fix `Parser::look_ahead``Parser::look_ahead` has a slow but simple general case, and a fast special case that is hit most of the time. But the special case is buggy and behaves differently to the general case. There are also no unit tests. This PR fixes all of this, resulting in a `Parser::look_ahead` that is equally fast, slightly simpler, more correct, and better tested.r? `@davidtwco`
2 parents5d76a13 +100f3fd commit62c068f

File tree

2 files changed

+147
-30
lines changed

2 files changed

+147
-30
lines changed

‎compiler/rustc_parse/src/parser/mod.rs‎

Lines changed: 26 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,41 +1118,37 @@ impl<'a> Parser<'a> {
11181118
returnlooker(&self.token);
11191119
}
11201120

1121-
ifletSome(&(_, span, _, delim)) =self.token_cursor.stack.last()
1122-
&& delim !=Delimiter::Invisible
1123-
{
1124-
// We are not in the outermost token stream, and the token stream
1125-
// we are in has non-skipped delimiters. Look for skipped
1126-
// delimiters in the lookahead range.
1127-
let tree_cursor =&self.token_cursor.tree_cursor;
1128-
let all_normal =(0..dist).all(|i|{
1129-
let token = tree_cursor.look_ahead(i);
1130-
!matches!(token,Some(TokenTree::Delimited(..,Delimiter::Invisible, _)))
1131-
});
1132-
if all_normal{
1133-
// There were no skipped delimiters. Do lookahead by plain indexing.
1134-
returnmatch tree_cursor.look_ahead(dist -1){
1135-
Some(tree) =>{
1136-
// Indexing stayed within the current token stream.
1137-
match tree{
1138-
TokenTree::Token(token, _) =>looker(token),
1139-
TokenTree::Delimited(dspan, _, delim, _) =>{
1140-
looker(&Token::new(token::OpenDelim(*delim), dspan.open))
1141-
}
1121+
// Typically around 98% of the `dist > 0` cases have `dist == 1`, so we
1122+
// have a fast special case for that.
1123+
if dist ==1{
1124+
// The index is zero because the tree cursor's index always points
1125+
// to the next token to be gotten.
1126+
matchself.token_cursor.tree_cursor.look_ahead(0){
1127+
Some(tree) =>{
1128+
// Indexing stayed within the current token tree.
1129+
returnmatch tree{
1130+
TokenTree::Token(token, _) =>looker(token),
1131+
TokenTree::Delimited(dspan, _, delim, _) =>{
1132+
looker(&Token::new(token::OpenDelim(*delim), dspan.open))
11421133
}
1134+
};
1135+
}
1136+
None =>{
1137+
// The tree cursor lookahead went (one) past the end of the
1138+
// current token tree. Try to return a close delimiter.
1139+
ifletSome(&(_, span, _, delim)) =self.token_cursor.stack.last()
1140+
&& delim !=Delimiter::Invisible
1141+
{
1142+
// We are not in the outermost token stream, so we have
1143+
// delimiters. Also, those delimiters are not skipped.
1144+
returnlooker(&Token::new(token::CloseDelim(delim), span.close));
11431145
}
1144-
None =>{
1145-
// Indexing went past the end of the current token
1146-
// stream. Use the close delimiter, no matter how far
1147-
// ahead `dist` went.
1148-
looker(&Token::new(token::CloseDelim(delim), span.close))
1149-
}
1150-
};
1146+
}
11511147
}
11521148
}
11531149

1154-
//We are in a more complex case.Just clone the token cursor and use
1155-
//`next`, skipping delimiters asnecessary. Slow but simple.
1150+
// Just clone the token cursor and use `next`, skipping delimiters as
1151+
// necessary. Slow but simple.
11561152
letmut cursor =self.token_cursor.clone();
11571153
letmut i =0;
11581154
letmut token =Token::dummy();

‎compiler/rustc_parse/src/parser/tests.rs‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1376,6 +1376,127 @@ fn ttdelim_span() {
13761376
});
13771377
}
13781378

1379+
// Uses a macro rather than a function so that failure messages mention the
1380+
// correct line in the test function.
1381+
macro_rules! look{
1382+
($p:ident, $dist:literal, $kind:expr) =>{
1383+
$p.look_ahead($dist, |tok| assert_eq!($kind, tok.kind));
1384+
};
1385+
}
1386+
1387+
#[test]
1388+
fnlook_ahead(){
1389+
create_default_session_globals_then(||{
1390+
let sym_f =Symbol::intern("f");
1391+
let sym_x =Symbol::intern("x");
1392+
#[allow(non_snake_case)]
1393+
let sym_S =Symbol::intern("S");
1394+
let raw_no =IdentIsRaw::No;
1395+
1396+
let psess =psess();
1397+
letmut p =string_to_parser(&psess,"fn f(x: u32) { x } struct S;".to_string());
1398+
1399+
// Current position is the `fn`.
1400+
look!(p,0, token::Ident(kw::Fn, raw_no));
1401+
look!(p,1, token::Ident(sym_f, raw_no));
1402+
look!(p,2, token::OpenDelim(Delimiter::Parenthesis));
1403+
look!(p,3, token::Ident(sym_x, raw_no));
1404+
look!(p,4, token::Colon);
1405+
look!(p,5, token::Ident(sym::u32, raw_no));
1406+
look!(p,6, token::CloseDelim(Delimiter::Parenthesis));
1407+
look!(p,7, token::OpenDelim(Delimiter::Brace));
1408+
look!(p,8, token::Ident(sym_x, raw_no));
1409+
look!(p,9, token::CloseDelim(Delimiter::Brace));
1410+
look!(p,10, token::Ident(kw::Struct, raw_no));
1411+
look!(p,11, token::Ident(sym_S, raw_no));
1412+
look!(p,12, token::Semi);
1413+
// Any lookahead past the end of the token stream returns `Eof`.
1414+
look!(p,13, token::Eof);
1415+
look!(p,14, token::Eof);
1416+
look!(p,15, token::Eof);
1417+
look!(p,100, token::Eof);
1418+
1419+
// Move forward to the first `x`.
1420+
for _in0..3{
1421+
p.bump();
1422+
}
1423+
look!(p,0, token::Ident(sym_x, raw_no));
1424+
look!(p,1, token::Colon);
1425+
look!(p,2, token::Ident(sym::u32, raw_no));
1426+
look!(p,3, token::CloseDelim(Delimiter::Parenthesis));
1427+
look!(p,4, token::OpenDelim(Delimiter::Brace));
1428+
look!(p,5, token::Ident(sym_x, raw_no));
1429+
look!(p,6, token::CloseDelim(Delimiter::Brace));
1430+
look!(p,7, token::Ident(kw::Struct, raw_no));
1431+
look!(p,8, token::Ident(sym_S, raw_no));
1432+
look!(p,9, token::Semi);
1433+
look!(p,10, token::Eof);
1434+
look!(p,11, token::Eof);
1435+
look!(p,100, token::Eof);
1436+
1437+
// Move forward to the `;`.
1438+
for _in0..9{
1439+
p.bump();
1440+
}
1441+
look!(p,0, token::Semi);
1442+
// Any lookahead past the end of the token stream returns `Eof`.
1443+
look!(p,1, token::Eof);
1444+
look!(p,100, token::Eof);
1445+
1446+
// Move one past the `;`, i.e. past the end of the token stream.
1447+
p.bump();
1448+
look!(p,0, token::Eof);
1449+
look!(p,1, token::Eof);
1450+
look!(p,100, token::Eof);
1451+
1452+
// Bumping after Eof is idempotent.
1453+
p.bump();
1454+
look!(p,0, token::Eof);
1455+
look!(p,1, token::Eof);
1456+
look!(p,100, token::Eof);
1457+
});
1458+
}
1459+
1460+
/// There used to be some buggy behaviour when using `look_ahead` not within
1461+
/// the outermost token stream, which this test covers.
1462+
#[test]
1463+
fnlook_ahead_non_outermost_stream(){
1464+
create_default_session_globals_then(||{
1465+
let sym_f =Symbol::intern("f");
1466+
let sym_x =Symbol::intern("x");
1467+
#[allow(non_snake_case)]
1468+
let sym_S =Symbol::intern("S");
1469+
let raw_no =IdentIsRaw::No;
1470+
1471+
let psess =psess();
1472+
letmut p =string_to_parser(&psess,"mod m { fn f(x: u32) { x } struct S; }".to_string());
1473+
1474+
// Move forward to the `fn`, which is not within the outermost token
1475+
// stream (because it's inside the `mod { ... }`).
1476+
for _in0..3{
1477+
p.bump();
1478+
}
1479+
look!(p,0, token::Ident(kw::Fn, raw_no));
1480+
look!(p,1, token::Ident(sym_f, raw_no));
1481+
look!(p,2, token::OpenDelim(Delimiter::Parenthesis));
1482+
look!(p,3, token::Ident(sym_x, raw_no));
1483+
look!(p,4, token::Colon);
1484+
look!(p,5, token::Ident(sym::u32, raw_no));
1485+
look!(p,6, token::CloseDelim(Delimiter::Parenthesis));
1486+
look!(p,7, token::OpenDelim(Delimiter::Brace));
1487+
look!(p,8, token::Ident(sym_x, raw_no));
1488+
look!(p,9, token::CloseDelim(Delimiter::Brace));
1489+
look!(p,10, token::Ident(kw::Struct, raw_no));
1490+
look!(p,11, token::Ident(sym_S, raw_no));
1491+
look!(p,12, token::Semi);
1492+
look!(p,13, token::CloseDelim(Delimiter::Brace));
1493+
// Any lookahead past the end of the token stream returns `Eof`.
1494+
look!(p,14, token::Eof);
1495+
look!(p,15, token::Eof);
1496+
look!(p,100, token::Eof);
1497+
});
1498+
}
1499+
13791500
// This tests that when parsing a string (rather than a file) we don't try
13801501
// and read in a file for a module declaration and just parse a stub.
13811502
// See `recurse_into_file_modules` in the parser.

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp