Auto merge of rust-lang#133436 - nnethercote:rm-NtVis-NtTy, r=petroch…

…enkov

Remove `NtVis` and `NtTy`

The next part of rust-lang#124141. The first actual remove of `Nonterminal` variants. `NtVis` is a simple case that doesn't get much use, but `NtTy` is more complex.

r? `@petrochenkov`

compiler/rustc_ast/src/ast_traits.rs

@@ -203,10 +203,8 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtStmt(stmt) => stmt.tokens(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
             Nonterminal::NtPat(pat) => pat.tokens(),
-            Nonterminal::NtTy(ty) => ty.tokens(),
             Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
             Nonterminal::NtPath(path) => path.tokens(),
-            Nonterminal::NtVis(vis) => vis.tokens(),
             Nonterminal::NtBlock(block) => block.tokens(),
         }
     }
@@ -216,10 +214,8 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
             Nonterminal::NtPat(pat) => pat.tokens_mut(),
-            Nonterminal::NtTy(ty) => ty.tokens_mut(),
             Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
             Nonterminal::NtPath(path) => path.tokens_mut(),
-            Nonterminal::NtVis(vis) => vis.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),
         }
     }

compiler/rustc_ast/src/mut_visit.rs

@@ -907,7 +907,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
         }),
         token::NtPat(pat) => vis.visit_pat(pat),
         token::NtExpr(expr) => vis.visit_expr(expr),
-        token::NtTy(ty) => vis.visit_ty(ty),
         token::NtLiteral(expr) => vis.visit_expr(expr),
         token::NtMeta(item) => {
             let AttrItem { unsafety: _, path, args, tokens } = item.deref_mut();
@@ -916,7 +915,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
             visit_lazy_tts(vis, tokens);
         }
         token::NtPath(path) => vis.visit_path(path),
-        token::NtVis(visib) => vis.visit_vis(visib),
     }
 }
 

compiler/rustc_ast/src/token.rs

@@ -84,7 +84,9 @@ pub enum MetaVarKind {
         // This field is needed for `Token::can_begin_string_literal`.
         can_begin_string_literal: bool,
     },
-    Ty,
+    Ty {
+        is_path: bool,
+    },
     Ident,
     Lifetime,
     Literal,
@@ -104,7 +106,7 @@ impl fmt::Display for MetaVarKind {
             MetaVarKind::Pat(PatParam { inferred: false }) => sym::pat_param,
             MetaVarKind::Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
             MetaVarKind::Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
-            MetaVarKind::Ty => sym::ty,
+            MetaVarKind::Ty { .. } => sym::ty,
             MetaVarKind::Ident => sym::ident,
             MetaVarKind::Lifetime => sym::lifetime,
             MetaVarKind::Literal => sym::literal,
@@ -659,15 +661,14 @@ impl Token {
                     | NtMeta(..)
                     | NtPat(..)
                     | NtPath(..)
-                    | NtTy(..)
                 ),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                 MetaVarKind::Expr { .. } |
                 MetaVarKind::Literal |
                 MetaVarKind::Meta |
                 MetaVarKind::Pat(_) |
                 MetaVarKind::Path |
-                MetaVarKind::Ty
+                MetaVarKind::Ty { .. }
             ))) => true,
             _ => false,
         }
@@ -688,9 +689,9 @@ impl Token {
             Lifetime(..)                | // lifetime bound in trait object
             Lt | BinOp(Shl)             | // associated path
             PathSep                      => true, // global path
-            Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
+            Interpolated(ref nt) => matches!(&**nt, NtPath(..)),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
-                MetaVarKind::Ty |
+                MetaVarKind::Ty { .. } |
                 MetaVarKind::Path
             ))) => true,
             // For anonymous structs or unions, which only appear in specific positions
@@ -969,6 +970,15 @@ impl Token {
         }
     }
 
+    /// Is this an invisible open delimiter at the start of a token sequence
+    /// from an expanded metavar?
+    pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
+        match self.kind {
+            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
+            _ => None,
+        }
+    }
+
     pub fn glue(&self, joint: &Token) -> Option<Token> {
         let kind = match self.kind {
             Eq => match joint.kind {
@@ -1067,12 +1077,10 @@ pub enum Nonterminal {
     NtStmt(P<ast::Stmt>),
     NtPat(P<ast::Pat>),
     NtExpr(P<ast::Expr>),
-    NtTy(P<ast::Ty>),
     NtLiteral(P<ast::Expr>),
     /// Stuff inside brackets for attributes
     NtMeta(P<ast::AttrItem>),
     NtPath(P<ast::Path>),
-    NtVis(P<ast::Visibility>),
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -1166,10 +1174,8 @@ impl Nonterminal {
             NtStmt(stmt) => stmt.span,
             NtPat(pat) => pat.span,
             NtExpr(expr) | NtLiteral(expr) => expr.span,
-            NtTy(ty) => ty.span,
             NtMeta(attr_item) => attr_item.span(),
             NtPath(path) => path.span,
-            NtVis(vis) => vis.span,
         }
     }
 
@@ -1181,10 +1187,8 @@ impl Nonterminal {
             NtPat(..) => "pattern",
             NtExpr(..) => "expression",
             NtLiteral(..) => "literal",
-            NtTy(..) => "type",
             NtMeta(..) => "attribute",
             NtPath(..) => "path",
-            NtVis(..) => "visibility",
         }
     }
 }
@@ -1207,11 +1211,9 @@ impl fmt::Debug for Nonterminal {
             NtStmt(..) => f.pad("NtStmt(..)"),
             NtPat(..) => f.pad("NtPat(..)"),
             NtExpr(..) => f.pad("NtExpr(..)"),
-            NtTy(..) => f.pad("NtTy(..)"),
             NtLiteral(..) => f.pad("NtLiteral(..)"),
             NtMeta(..) => f.pad("NtMeta(..)"),
             NtPath(..) => f.pad("NtPath(..)"),
-            NtVis(..) => f.pad("NtVis(..)"),
         }
     }
 }

compiler/rustc_ast/src/tokenstream.rs

@@ -469,10 +469,8 @@ impl TokenStream {
             }
             Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
             Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
-            Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
             Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
             Nonterminal::NtPath(path) => TokenStream::from_ast(path),
-            Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
         }
     }

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -1,10 +1,13 @@
 use std::mem;
 use std::sync::Arc;
 
-use rustc_ast::ExprKind;
 use rustc_ast::mut_visit::{self, MutVisitor};
-use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, LitKind, Nonterminal, Token, TokenKind};
+use rustc_ast::token::{
+    self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
+    TokenKind,
+};
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
+use rustc_ast::{ExprKind, TyKind};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
 use rustc_parse::lexer::nfc_normalize;
@@ -274,6 +277,33 @@ pub(super) fn transcribe<'a>(
                 // some of the unnecessary whitespace.
                 let ident = MacroRulesNormalizedIdent::new(original_ident);
                 if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
+                    // We wrap the tokens in invisible delimiters, unless they are already wrapped
+                    // in invisible delimiters with the same `MetaVarKind`. Because some proc
+                    // macros can't multiple layers of invisible delimiters of the same
+                    // `MetaVarKind`. This loses some span info, though it hopefully won't matter.
+                    let mut mk_delimited = |mv_kind, mut stream: TokenStream| {
+                        if stream.len() == 1 {
+                            let tree = stream.iter().next().unwrap();
+                            if let TokenTree::Delimited(_, _, delim, inner) = tree
+                                && let Delimiter::Invisible(InvisibleOrigin::MetaVar(mvk)) = delim
+                                && mv_kind == *mvk
+                            {
+                                stream = inner.clone();
+                            }
+                        }
+
+                        // Emit as a token stream within `Delimiter::Invisible` to maintain
+                        // parsing priorities.
+                        marker.visit_span(&mut sp);
+                        // Both the open delim and close delim get the same span, which covers the
+                        // `$foo` in the decl macro RHS.
+                        TokenTree::Delimited(
+                            DelimSpan::from_single(sp),
+                            DelimSpacing::new(Spacing::Alone, Spacing::Alone),
+                            Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)),
+                            stream,
+                        )
+                    };
                     let tt = match cur_matched {
                         MatchedSingle(ParseNtResult::Tt(tt)) => {
                             // `tt`s are emitted into the output stream directly as "raw tokens",
@@ -292,6 +322,13 @@ pub(super) fn transcribe<'a>(
                             let kind = token::NtLifetime(*ident, *is_raw);
                             TokenTree::token_alone(kind, sp)
                         }
+                        MatchedSingle(ParseNtResult::Ty(ty)) => {
+                            let is_path = matches!(&ty.kind, TyKind::Path(None, _path));
+                            mk_delimited(MetaVarKind::Ty { is_path }, TokenStream::from_ast(ty))
+                        }
+                        MatchedSingle(ParseNtResult::Vis(vis)) => {
+                            mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis))
+                        }
                         MatchedSingle(ParseNtResult::Nt(nt)) => {
                             // Other variables are emitted into the output stream as groups with
                             // `Delimiter::Invisible` to maintain parsing priorities.

compiler/rustc_parse/src/parser/mod.rs

@@ -117,12 +117,15 @@ macro_rules! maybe_recover_from_interpolated_ty_qpath {
     ($self: expr, $allow_qpath_recovery: expr) => {
         if $allow_qpath_recovery
             && $self.may_recover()
-            && $self.look_ahead(1, |t| t == &token::PathSep)
-            && let token::Interpolated(nt) = &$self.token.kind
-            && let token::NtTy(ty) = &**nt
+            && let Some(mv_kind) = $self.token.is_metavar_seq()
+            && let token::MetaVarKind::Ty { .. } = mv_kind
+            && $self.check_noexpect_past_close_delim(&token::PathSep)
         {
-            let ty = ty.clone();
-            $self.bump();
+            // Reparse the type, then move to recovery.
+            let ty = $self
+                .eat_metavar_seq(mv_kind, |this| this.parse_ty_no_question_mark_recover())
+                .expect("metavar seq ty");
+
             return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
         }
     };
@@ -614,6 +617,24 @@ impl<'a> Parser<'a> {
         self.token == *tok
     }
 
+    // Check the first token after the delimiter that closes the current
+    // delimited sequence. (Panics if used in the outermost token stream, which
+    // has no delimiters.) It uses a clone of the relevant tree cursor to skip
+    // past the entire `TokenTree::Delimited` in a single step, avoiding the
+    // need for unbounded token lookahead.
+    //
+    // Primarily used when `self.token` matches
+    // `OpenDelim(Delimiter::Invisible(_))`, to look ahead through the current
+    // metavar expansion.
+    fn check_noexpect_past_close_delim(&self, tok: &TokenKind) -> bool {
+        let mut tree_cursor = self.token_cursor.stack.last().unwrap().clone();
+        tree_cursor.bump();
+        matches!(
+            tree_cursor.curr(),
+            Some(TokenTree::Token(token::Token { kind, .. }, _)) if kind == tok
+        )
+    }
+
     /// Consumes a token 'tok' if it exists. Returns whether the given token was present.
     ///
     /// the main purpose of this function is to reduce the cluttering of the suggestions list
@@ -721,6 +742,43 @@ impl<'a> Parser<'a> {
         if !self.eat_keyword(exp) { self.unexpected() } else { Ok(()) }
     }
 
+    /// Consume a sequence produced by a metavar expansion, if present.
+    fn eat_metavar_seq<T>(
+        &mut self,
+        mv_kind: MetaVarKind,
+        f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
+    ) -> Option<T> {
+        self.eat_metavar_seq_with_matcher(|mvk| mvk == mv_kind, f)
+    }
+
+    /// A slightly more general form of `eat_metavar_seq`, for use with the
+    /// `MetaVarKind` variants that have parameters, where an exact match isn't
+    /// desired.
+    fn eat_metavar_seq_with_matcher<T>(
+        &mut self,
+        match_mv_kind: impl Fn(MetaVarKind) -> bool,
+        mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
+    ) -> Option<T> {
+        if let token::OpenDelim(delim) = self.token.kind
+            && let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
+            && match_mv_kind(mv_kind)
+        {
+            self.bump();
+            let res = f(self).expect("failed to reparse {mv_kind:?}");
+            if let token::CloseDelim(delim) = self.token.kind
+                && let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
+                && match_mv_kind(mv_kind)
+            {
+                self.bump();
+                Some(res)
+            } else {
+                panic!("no close delim when reparsing {mv_kind:?}");
+            }
+        } else {
+            None
+        }
+    }
+
     /// Is the given keyword `kw` followed by a non-reserved identifier?
     fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool {
         self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())
@@ -1455,7 +1513,11 @@ impl<'a> Parser<'a> {
     /// so emit a proper diagnostic.
     // Public for rustfmt usage.
     pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
-        maybe_whole!(self, NtVis, |vis| vis.into_inner());
+        if let Some(vis) = self
+            .eat_metavar_seq(MetaVarKind::Vis, |this| this.parse_visibility(FollowedByType::Yes))
+        {
+            return Ok(vis);
+        }
 
         if !self.eat_keyword(exp!(Pub)) {
             // We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
@@ -1683,7 +1745,9 @@ pub enum ParseNtResult {
     Tt(TokenTree),
     Ident(Ident, IdentIsRaw),
     Lifetime(Ident, IdentIsRaw),
+    Ty(P<ast::Ty>),
+    Vis(P<ast::Visibility>),
 
-    /// This case will eventually be removed, along with `Token::Interpolate`.
+    /// This variant will eventually be removed, along with `Token::Interpolate`.
     Nt(Arc<Nonterminal>),
 }