@@ -681,22 +681,40 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
681681}
682682}
683683
684+ // The easiest way to implement token stream pretty printing would be to
685+ // print each token followed by a single space. But that would produce ugly
686+ // output, so we go to some effort to do better.
687+ //
688+ // First, we track whether each token that appears in source code is
689+ // followed by a space, with `Spacing`, and reproduce that in the output.
690+ // This works well in a lot of cases. E.g. `stringify!(x + y)` produces
691+ // "x + y" and `stringify!(x+y)` produces "x+y".
692+ //
693+ // But this doesn't work for code produced by proc macros (which have no
694+ // original source text representation) nor for code produced by decl
695+ // macros (which are tricky because the whitespace after tokens appearing
696+ // in macro rules isn't always what you want in the produced output). For
697+ // these we mostly use `Spacing::Alone`, which is the conservative choice.
698+ //
699+ // So we have a backup mechanism for when `Spacing::Alone` occurs between a
700+ // pair of tokens: we check if that pair of tokens can obviously go
701+ // together without a space between them. E.g. token `x` followed by token
702+ // `,` is better printed as `x,` than `x ,`. (Even if the original source
703+ // code was `x ,`.)
704+ //
705+ // Finally, we must be careful about changing the output. Token pretty
706+ // printing is used by `stringify!` and `impl Display for
707+ // proc_macro::TokenStream`, and some programs rely on the output having a
708+ // particular form, even though they shouldn't. In particular, some proc
709+ // macros do `format!({stream})` on a token stream and then "parse" the
710+ // output with simple string matching that can't handle whitespace changes.
711+ // E.g. we have seen cases where a proc macro can handle `a :: b` but not
712+ // `a::b`. See #117433 for some examples.
684713fn print_tts ( & mut self , tts : & TokenStream , convert_dollar_crate : bool ) {
685714let mut iter = tts. trees ( ) . peekable ( ) ;
686715while let Some ( tt) = iter. next ( ) {
687716let spacing =self . print_tt ( tt, convert_dollar_crate) ;
688717if let Some ( next) = iter. peek ( ) {
689- // Should we print a space after `tt`? There are two guiding
690- // factors.
691- // - `spacing` is the more important and accurate one. Most
692- // tokens have good spacing information, and
693- // `Joint`/`JointHidden` get used a lot.
694- // - `space_between` is the backup. Code produced by proc
695- // macros has worse spacing information, with no
696- // `JointHidden` usage and too much `Alone` usage, which
697- // would result in over-spaced output such as
698- // `( x () , y . z )`. `space_between` avoids some of the
699- // excess whitespace.
700718if spacing ==Spacing :: Alone &&space_between ( tt, next) {
701719self . space ( ) ;
702720}
