Rollup of 7 perf-sensitive pull requests

Cargo.lock

@@ -4662,7 +4662,6 @@ dependencies = [
  "rustc_ast_lowering",
  "rustc_attr_ir",
  "rustc_attr_parsing",
- "rustc_crate_store",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_feature",
@@ -4787,6 +4786,7 @@ dependencies = [
  "rustc_ast_pretty",
  "rustc_attr_ir",
  "rustc_attr_parsing",
+ "rustc_crate_store",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_expand",
@@ -4906,6 +4906,7 @@ dependencies = [
  "rustc_data_structures",
  "rustc_hashes",
  "rustc_hir",
+ "rustc_macros",
  "rustc_middle",
  "rustc_session",
  "rustc_span",

compiler/rustc_ast/src/ast.rs

@@ -40,7 +40,7 @@ use thin_vec::{ThinVec, thin_vec};
 use crate::attr::data_structures::CfgEntry;
 pub use crate::format::*;
 use crate::token::{self, CommentKind, Delimiter};
-use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream, TokenTree};
+use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream, TokenTree};
 use crate::util::parser::{ExprPrecedence, Fixity};
 use crate::visit::{AssocCtxt, BoundKind, LifetimeCtxt};
 
@@ -1221,7 +1221,7 @@ impl Stmt {
     /// a trailing semicolon.
     ///
     /// This only modifies the parsed AST struct, not the attached
-    /// `LazyAttrTokenStream`. The parser is responsible for calling
+    /// `LazyTokenStream`. The parser is responsible for calling
     /// `ToAttrTokenStream::add_trailing_semi` when there is actually
     /// a semicolon in the tokenstream.
     pub fn add_trailing_semicolon(mut self) -> Self {
@@ -1281,7 +1281,7 @@ pub struct MacCallStmt {
     pub mac: Box<MacCall>,
     pub style: MacStmtStyle,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<LazyTokenStream>,
 }
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, Walkable)]
@@ -1308,7 +1308,7 @@ pub struct Local {
     pub span: Span,
     pub colon_sp: Option<Span>,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<LazyTokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, Walkable)]
@@ -1409,7 +1409,7 @@ pub struct Expr {
     pub kind: ExprKind,
     pub span: Span,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<LazyTokenStream>,
 }
 
 impl Expr {
@@ -3605,7 +3605,7 @@ impl GeneratedAttr {
         }
     }
 
-    pub fn token_trees(&self) -> Vec<TokenTree> {
+    pub fn append_token_trees_to(&self, _res: &mut Vec<TokenTree>) {
         unimplemented!(
             "generated attribute `{}` cannot re-enter the parser at this time",
             self.descr()
@@ -3618,7 +3618,7 @@ pub struct NormalAttr {
     pub item: AttrItem,
     // Tokens for the full attribute, e.g. `#[foo]`, `#![bar]`. (Compare this with
     // `ParseNtResult::Meta`; `expand_cfg_attr_item` is where the two cases interact.)
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<LazyTokenStream>,
 }
 
 impl NormalAttr {
@@ -3889,7 +3889,7 @@ pub struct Item<K = ItemKind> {
     ///
     /// Note that the tokens here do not include the outer attributes, but will
     /// include inner attributes.
-    pub tokens: Option<LazyAttrTokenStream>,
+    pub tokens: Option<LazyTokenStream>,
 }
 
 impl Item {

compiler/rustc_ast/src/ast_traits.rs

@@ -5,7 +5,7 @@
 use std::fmt;
 use std::marker::PhantomData;
 
-use crate::tokenstream::{LazyAttrTokenStream, WithTokens};
+use crate::tokenstream::{LazyTokenStream, WithTokens};
 use crate::{
     Arm, AssocItem, AttrItem, AttrKind, AttrVec, Attribute, Block, Crate, Expr, ExprField,
     FieldDef, ForeignItem, GenericParam, Item, NodeId, Param, Pat, PatField, Path, Stmt, StmtKind,
@@ -63,18 +63,18 @@ impl<T: HasNodeId> HasNodeId for Box<T> {
 
 /// A trait for AST nodes having (or not having) collected tokens.
 pub trait HasTokens: HasAttrs {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream>;
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>>;
+    fn tokens(&self) -> Option<&LazyTokenStream>;
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
 }
 
 macro_rules! impl_has_tokens {
     ($($T:ty),+ $(,)?) => {
         $(
             impl HasTokens for $T {
-                fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+                fn tokens(&self) -> Option<&LazyTokenStream> {
                     self.tokens.as_ref()
                 }
-                fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+                fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
                     Some(&mut self.tokens)
                 }
             }
@@ -86,10 +86,10 @@ macro_rules! impl_has_tokens_none {
     ($($T:ty),+ $(,)?) => {
         $(
             impl HasTokens for $T {
-                fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+                fn tokens(&self) -> Option<&LazyTokenStream> {
                     None
                 }
-                fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+                fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
                     None
                 }
             }
@@ -110,34 +110,34 @@ impl_has_tokens_none!(
 );
 
 impl<T: HasAttrs> HasTokens for WithTokens<T> {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&LazyTokenStream> {
         self.tokens.as_ref()
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
         Some(&mut self.tokens)
     }
 }
 
 impl<T: HasTokens> HasTokens for Option<T> {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&LazyTokenStream> {
         self.as_ref().and_then(|inner| inner.tokens())
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
         self.as_mut().and_then(|inner| inner.tokens_mut())
     }
 }
 
 impl<T: HasTokens> HasTokens for Box<T> {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&LazyTokenStream> {
         (**self).tokens()
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
         (**self).tokens_mut()
     }
 }
 
 impl HasTokens for StmtKind {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&LazyTokenStream> {
         match self {
             StmtKind::Let(local) => local.tokens.as_ref(),
             StmtKind::Item(item) => item.tokens(),
@@ -146,7 +146,7 @@ impl HasTokens for StmtKind {
             StmtKind::MacCall(mac) => mac.tokens.as_ref(),
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
         match self {
             StmtKind::Let(local) => Some(&mut local.tokens),
             StmtKind::Item(item) => item.tokens_mut(),
@@ -158,24 +158,24 @@ impl HasTokens for StmtKind {
 }
 
 impl HasTokens for Stmt {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&LazyTokenStream> {
         self.kind.tokens()
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
         self.kind.tokens_mut()
     }
 }
 
 impl HasTokens for Attribute {
-    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
+    fn tokens(&self) -> Option<&LazyTokenStream> {
         match &self.kind {
             AttrKind::Normal(normal) => normal.tokens.as_ref(),
             AttrKind::Synthetic(..) | AttrKind::DocComment(..) | AttrKind::Generated(..) => {
                 unreachable!()
             }
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
         Some(match &mut self.kind {
             AttrKind::Normal(normal) => &mut normal.tokens,
             AttrKind::Synthetic(..) | AttrKind::DocComment(..) | AttrKind::Generated(..) => {

compiler/rustc_ast/src/attr/mod.rs

@@ -20,8 +20,7 @@ use crate::token::{
     self, CommentKind, Delimiter, DocFragmentKind, InvisibleOrigin, MetaVarKind, Token,
 };
 use crate::tokenstream::{
-    AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, LazyAttrTokenStream, Spacing,
-    TokenStream, TokenStreamIter, TokenTree,
+    DelimSpacing, DelimSpan, LazyTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree,
 };
 use crate::util::comments;
 use crate::util::literal::escape_string_symbol;
@@ -306,21 +305,20 @@ impl Attribute {
         }
     }
 
-    pub fn token_trees(&self) -> Vec<TokenTree> {
+    pub fn append_token_trees_to(&self, res: &mut Vec<TokenTree>) {
         match &self.kind {
             AttrKind::Normal(normal) => normal
                 .tokens
                 .as_ref()
                 .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}"))
-                .to_attr_token_stream()
-                .to_token_trees(),
+                .append_token_trees_to(res),
             // Empty tokens here ensures synthetic attributes are invisible to proc macros.
-            AttrKind::Synthetic(..) => vec![],
-            AttrKind::Generated(generated) => generated.token_trees(),
-            AttrKind::DocComment(comment_kind, data) => vec![TokenTree::token_alone(
+            AttrKind::Synthetic(..) => {}
+            AttrKind::Generated(generated) => generated.append_token_trees_to(res),
+            AttrKind::DocComment(comment_kind, data) => res.push(TokenTree::token_alone(
                 token::DocComment(*comment_kind, self.style, *data),
                 self.span,
-            )],
+            )),
         }
     }
 
@@ -539,7 +537,7 @@ impl MetaItem {
                 unreachable!()
             }
             Some(TokenTree::Token(Token { kind, .. }, _)) if kind.is_delim() => {
-                panic!("Should be `AttrTokenTree::Delimited`, not delim tokens: {:?}", tt);
+                panic!("Should be `TokenTree::Delimited`, not delim tokens: {:?}", tt);
             }
             _ => return None,
         };
@@ -750,7 +748,7 @@ pub fn mk_doc_comment(
 pub fn mk_attr_from_item(
     g: &AttrIdGenerator,
     item: AttrItem,
-    tokens: Option<LazyAttrTokenStream>,
+    tokens: Option<LazyTokenStream>,
     style: AttrStyle,
     span: Span,
 ) -> Attribute {
@@ -762,28 +760,24 @@ pub fn mk_attr_from_item(
     }
 }
 
-fn mk_attr_tokens(
-    style: AttrStyle,
-    item_tokens: AttrTokenStream,
-    span: Span,
-) -> LazyAttrTokenStream {
+fn mk_attr_tokens(style: AttrStyle, item_tokens: TokenStream, span: Span) -> LazyTokenStream {
     let mut tokens = match style {
         AttrStyle::Outer => {
-            vec![AttrTokenTree::Token(Token::new(token::Pound, span), Spacing::JointHidden)]
+            vec![TokenTree::Token(Token::new(token::Pound, span), Spacing::JointHidden)]
         }
         AttrStyle::Inner => vec![
-            AttrTokenTree::Token(Token::new(token::Pound, span), Spacing::Joint),
-            AttrTokenTree::Token(Token::new(token::Bang, span), Spacing::JointHidden),
+            TokenTree::Token(Token::new(token::Pound, span), Spacing::Joint),
+            TokenTree::Token(Token::new(token::Bang, span), Spacing::JointHidden),
         ],
     };
-    tokens.push(AttrTokenTree::Delimited(
+    tokens.push(TokenTree::Delimited(
         DelimSpan::from_single(span),
         DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
         Delimiter::Bracket,
         item_tokens,
     ));
 
-    LazyAttrTokenStream::new_direct(AttrTokenStream::new(tokens))
+    LazyTokenStream::new_direct(TokenStream::new(tokens))
 }
 
 // `span` is used for the `Attribute` and everything within it (except for any span within
@@ -794,7 +788,7 @@ pub fn mk_attr_word(g: &AttrIdGenerator, style: AttrStyle, name: Symbol, span: S
 
     let tokens = Some(mk_attr_tokens(
         style,
-        AttrTokenStream::new(vec![AttrTokenTree::Token(
+        TokenStream::new(vec![TokenTree::Token(
             Token::from_ast_ident(Ident::new(name, span)),
             Spacing::Alone,
         )]),
@@ -833,13 +827,13 @@ pub fn mk_attr_nested_word(
 
     let tokens = Some(mk_attr_tokens(
         style,
-        AttrTokenStream::new(vec![
-            AttrTokenTree::Token(Token::from_ast_ident(Ident::new(outer, span)), Spacing::Alone),
-            AttrTokenTree::Delimited(
+        TokenStream::new(vec![
+            TokenTree::Token(Token::from_ast_ident(Ident::new(outer, span)), Spacing::Alone),
+            TokenTree::Delimited(
                 DelimSpan::from_single(span),
                 DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
                 Delimiter::Parenthesis,
-                AttrTokenStream::new(vec![AttrTokenTree::Token(
+                TokenStream::new(vec![TokenTree::Token(
                     Token::from_ast_ident(Ident::new(inner, span)),
                     Spacing::Alone,
                 )]),
@@ -879,10 +873,10 @@ pub fn mk_attr_name_value_str(
 
     let tokens = Some(mk_attr_tokens(
         style,
-        AttrTokenStream::new(vec![
-            AttrTokenTree::Token(Token::from_ast_ident(Ident::new(name, span)), Spacing::Alone),
-            AttrTokenTree::Token(Token::new(token::Eq, span), Spacing::Alone),
-            AttrTokenTree::Token(
+        TokenStream::new(vec![
+            TokenTree::Token(Token::from_ast_ident(Ident::new(name, span)), Spacing::Alone),
+            TokenTree::Token(Token::new(token::Eq, span), Spacing::Alone),
+            TokenTree::Token(
                 Token::new(token::TokenKind::lit(lit.kind, lit.symbol, lit.suffix), span),
                 Spacing::Alone,
             ),