Skip to content

Commit

Permalink
XXX: finalizing
Browse files Browse the repository at this point in the history
  • Loading branch information
@nnethercote
nnethercote committed Jun 23, 2024
1 parent ab3a0f6 commit c745726
Show file tree
Hide file tree
Showing 17 changed files with 472 additions and 443 deletions.
4 changes: 2 additions & 2 deletions compiler/rustc_ast/src/attr/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit};
use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, InvisibleOrigin, NonterminalKind, Token};
use crate::token::{self, CommentKind, Delimiter, InvisibleOrigin, MetaVarKind, Token};
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
use crate::util::comments;
Expand Down Expand Up @@ -368,7 +368,7 @@ impl MetaItem {
_span,
_spacing,
Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Meta | NonterminalKind::Path,
MetaVarKind::Meta | MetaVarKind::Path,
)),
_stream,
)) => {
Expand Down
213 changes: 128 additions & 85 deletions compiler/rustc_ast/src/token.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
pub use BinOpToken::*;
pub use LitKind::*;
pub use NtExprKind::*;
pub use NtPatKind::*;
pub use TokenKind::*;

use crate::ast;
Expand Down Expand Up @@ -38,7 +40,7 @@ pub enum BinOpToken {
#[derive(Copy, Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum InvisibleOrigin {
// From the expansion of a metavariable in a declarative macro.
MetaVar(NonterminalKind),
MetaVar(MetaVarKind),

// Converted from `proc_macro::Delimiter` in
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
Expand All @@ -49,6 +51,54 @@ pub enum InvisibleOrigin {
FlattenToken,
}

/// Annoyingly similar to `NonterminalKind`, but the slightly differences are important.
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum MetaVarKind {
Item,
Block,
Stmt,
Pat(NtPatKind),
Expr {
kind: NtExprKind,
// This field is needed for `Token::can_begin_literal_maybe_minus`.
can_begin_literal_maybe_minus: bool,
// This field is needed for `Token::can_begin_string_literal`.
can_begin_string_literal: bool,
},
Ty,
Ident,
Lifetime,
Literal,
Meta,
Path,
Vis,
TT,
}

impl fmt::Display for MetaVarKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use MetaVarKind::*;
let sym = match self {
Item => sym::item,
Block => sym::block,
Stmt => sym::stmt,
Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
Pat(PatParam { inferred: false }) => sym::pat_param,
Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
Ty => sym::ty,
Ident => sym::ident,
Lifetime => sym::lifetime,
Literal => sym::literal,
Meta => sym::meta,
Path => sym::path,
Vis => sym::vis,
TT => sym::tt,
};
write!(f, "{}", sym)
}
}

/// Describes how a sequence of token trees is delimited.
/// Cannot use `proc_macro::Delimiter` directly because this
/// structure should implement some additional traits.
Expand Down Expand Up @@ -136,15 +186,17 @@ impl Lit {
match token.uninterpolate().kind {
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
Literal(token_lit) => Some(token_lit),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Literal))) => {
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(MetaVarKind::Literal))) => {
panic!("njn: FROM_TOKEN (1)");
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
// {
// Some(token_lit)
// }
}
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Expr))) => {
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(MetaVarKind::Expr {
..
}))) => {
panic!("njn: FROM_TOKEN (2)");
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
Expand Down Expand Up @@ -509,11 +561,10 @@ impl Token {
Lifetime(..) | // labeled loop
Pound => true, // expression attributes
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Block |
NonterminalKind::Expr |
NonterminalKind::Expr2021 { .. } |
NonterminalKind::Literal |
NonterminalKind::Path
MetaVarKind::Block |
MetaVarKind::Expr { .. } |
MetaVarKind::Literal |
MetaVarKind::Path
))) => true,
_ => false,
}
Expand All @@ -536,11 +587,10 @@ impl Token {
| Lt | BinOp(Shl) // associated path
| PathSep => true, // global path
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Block |
NonterminalKind::PatParam { .. } |
NonterminalKind::PatWithOr |
NonterminalKind::Path |
NonterminalKind::Literal
MetaVarKind::Block |
MetaVarKind::Pat(_) |
MetaVarKind::Path |
MetaVarKind::Literal
))) => true,
_ => false,
}
Expand All @@ -562,8 +612,8 @@ impl Token {
Lt | BinOp(Shl) | // associated path
PathSep => true, // global path
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Ty |
NonterminalKind::Path
MetaVarKind::Ty |
MetaVarKind::Path
))) => true,
// For anonymous structs or unions, which only appear in specific positions
// (type of struct fields or union fields), we don't consider them as regular types
Expand All @@ -577,10 +627,7 @@ impl Token {
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Expr
| NonterminalKind::Expr2021 { .. }
| NonterminalKind::Block
| NonterminalKind::Literal,
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
))) => true,
_ => false,
}
Expand Down Expand Up @@ -628,42 +675,25 @@ impl Token {
match self.uninterpolate().kind {
Literal(..) | BinOp(Minus) => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
// njn: fix up
// Interpolated(ref nt) => match &**nt {
// NtLiteral(_) => true,
// NtExpr(e) => match &e.kind {
// ast::ExprKind::Lit(_) => true,
// ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
// matches!(&e.kind, ast::ExprKind::Lit(_))
// }
// _ => false,
// },
// _ => false,
// },
// njn: too simple compared to what's above?
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Literal | NonterminalKind::Expr | NonterminalKind::Expr2021 { .. },
))) => true,
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
MetaVarKind::Literal => true,
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
can_begin_literal_maybe_minus
}
_ => false,
},
_ => false,
}
}

pub fn can_begin_string_literal(&self) -> bool {
match self.uninterpolate().kind {
Literal(..) => true,
// njn: fix up
// Interpolated(ref nt) => match &**nt {
// NtLiteral(_) => true,
// NtExpr(e) => match &e.kind {
// ast::ExprKind::Lit(_) => true,
// _ => false,
// },
// _ => false,
// },
// njn: too simple compared to what's above?
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
NonterminalKind::Literal | NonterminalKind::Expr | NonterminalKind::Expr2021,
))) => true,
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
MetaVarKind::Literal => true,
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
_ => false,
},
_ => false,
}
}
Expand Down Expand Up @@ -718,24 +748,24 @@ impl Token {
self.ident().is_some_and(|(ident, _)| ident.name == name)
}

/// Would `maybe_reparse_metavar_expr` in `parser.rs` return `Ok(..)`?
/// Would `eat_metavar_expr` in `parser.rs` return `Ok(..)`?
/// That is, is this a pre-parsed expression dropped into the token stream
/// (which happens while parsing the result of macro expansion)?
pub fn is_metavar_expr(&self) -> bool {
matches!(
self.is_metavar_seq(),
Some(
NonterminalKind::Expr
| NonterminalKind::Literal
| NonterminalKind::Path
| NonterminalKind::Block
MetaVarKind::Expr { .. }
| MetaVarKind::Literal
| MetaVarKind::Path
| MetaVarKind::Block
)
)
}

/// Are we at a block from a metavar (`$b:block`)?
pub fn is_metavar_block(&self) -> bool {
matches!(self.is_metavar_seq(), Some(NonterminalKind::Block))
matches!(self.is_metavar_seq(), Some(MetaVarKind::Block))
}

/// Returns `true` if the token is either the `mut` or `const` keyword.
Expand All @@ -750,7 +780,7 @@ impl Token {
pub fn is_path_start(&self) -> bool {
self == &PathSep
|| self.is_qpath_start()
|| matches!(self.is_metavar_seq(), Some(NonterminalKind::Path))
|| matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
|| self.is_path_segment_keyword()
|| self.is_ident() && !self.is_reserved_ident()
}
Expand Down Expand Up @@ -822,7 +852,7 @@ impl Token {

/// Is this an invisible open delimiter at the start of a token sequence
/// from an expanded metavar?
pub fn is_metavar_seq(&self) -> Option<NonterminalKind> {
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
match self.kind {
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
_ => None,
Expand Down Expand Up @@ -898,24 +928,34 @@ impl PartialEq<TokenKind> for Token {
}
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum NtPatKind {
// Matches or-patterns. Was written using `pat` in edition 2021 or later.
PatWithOr,
// Doesn't match or-patterns.
// - `inferred`: was written using `pat` in edition 2015 or 2018.
// - `!inferred`: was written using `pat_param`.
PatParam { inferred: bool },
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum NtExprKind {
// Matches expressions using the post-edition 2024. Was written using
// `expr` in edition 2024 or later.
Expr,
// Matches expressions using the pre-edition 2024 rules.
// - `inferred`: was written using `expr` in edition 2021 or earlier.
// - `!inferred`: was written using `expr_2021`.
Expr2021 { inferred: bool },
}

#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum NonterminalKind {
Item,
Block,
Stmt,
PatParam {
/// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the
/// edition of the span. This is used for diagnostics.
inferred: bool,
},
PatWithOr,
Expr,
/// Matches an expression using the rules from edition 2021 and earlier.
Expr2021 {
/// Keep track of whether the user used `:expr` or `:expr_2021` and we inferred it from the
/// edition of the span. This is used for diagnostics AND feature gating.
inferred: bool,
},
Pat(NtPatKind),
Expr(NtExprKind),
Ty,
Ident,
Lifetime,
Expand All @@ -937,20 +977,22 @@ impl NonterminalKind {
sym::item => NonterminalKind::Item,
sym::block => NonterminalKind::Block,
sym::stmt => NonterminalKind::Stmt,
sym::pat => match edition() {
Edition::Edition2015 | Edition::Edition2018 => {
NonterminalKind::PatParam { inferred: true }
sym::pat => {
if edition().at_least_rust_2021() {
NonterminalKind::Pat(PatWithOr)
} else {
NonterminalKind::Pat(PatParam { inferred: true })
}
Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr,
},
sym::pat_param => NonterminalKind::PatParam { inferred: false },
sym::expr => match edition() {
Edition::Edition2015 | Edition::Edition2018 | Edition::Edition2021 => {
NonterminalKind::Expr2021 { inferred: true }
}
sym::pat_param => NonterminalKind::Pat(PatParam { inferred: false }),
sym::expr => {
if edition().at_least_rust_2024() {
NonterminalKind::Expr(Expr)
} else {
NonterminalKind::Expr(Expr2021 { inferred: true })
}
Edition::Edition2024 => NonterminalKind::Expr,
},
sym::expr_2021 => NonterminalKind::Expr2021 { inferred: false },
}
sym::expr_2021 => NonterminalKind::Expr(Expr2021 { inferred: false }),
sym::ty => NonterminalKind::Ty,
sym::ident => NonterminalKind::Ident,
sym::lifetime => NonterminalKind::Lifetime,
Expand All @@ -962,15 +1004,16 @@ impl NonterminalKind {
_ => return None,
})
}

fn symbol(self) -> Symbol {
match self {
NonterminalKind::Item => sym::item,
NonterminalKind::Block => sym::block,
NonterminalKind::Stmt => sym::stmt,
NonterminalKind::PatParam { inferred: false } => sym::pat_param,
NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat,
NonterminalKind::Expr | NonterminalKind::Expr2021 { inferred: true } => sym::expr,
NonterminalKind::Expr2021 { inferred: false } => sym::expr_2021,
NonterminalKind::Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
NonterminalKind::Pat(PatParam { inferred: false }) => sym::pat_param,
NonterminalKind::Expr(Expr2021 { inferred: true } | Expr) => sym::expr,
NonterminalKind::Expr(Expr2021 { inferred: false }) => sym::expr_2021,
NonterminalKind::Ty => sym::ty,
NonterminalKind::Ident => sym::ident,
NonterminalKind::Lifetime => sym::lifetime,
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_builtin_macros/src/cfg_eval.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ impl CfgEval<'_, '_> {
},
Annotatable::Stmt(_) => |parser| {
Ok(Annotatable::Stmt(P(parser
.parse_stmt_without_recovery(false, ForceCollect::Yes)?
.parse_stmt_without_recovery(false, ForceCollect::Yes, false)?
.unwrap())))
},
Annotatable::Expr(_) => {
Expand Down
Loading

0 comments on commit c745726

Please sign in to comment.