Stop generating an intermediate deref when taking references to statics

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -537,10 +537,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout };
                 (bx, operand)
             }
-            mir::Rvalue::ThreadLocalRef(def_id) => {
+            mir::Rvalue::ThreadLocalRef(def_id, ty) => {
                 assert!(bx.cx().tcx().is_static(def_id));
                 let static_ = bx.get_static(def_id);
-                let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id));
+                let layout = bx.layout_of(ty);
                 let operand = OperandRef::from_immediate_or_packed_pair(&mut bx, static_, layout);
                 (bx, operand)
             }
@@ -767,7 +767,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::Rvalue::UnaryOp(..) |
             mir::Rvalue::Discriminant(..) |
             mir::Rvalue::NullaryOp(..) |
-            mir::Rvalue::ThreadLocalRef(_) |
+            mir::Rvalue::ThreadLocalRef(..) |
             mir::Rvalue::Use(..) => // (*)
                 true,
             mir::Rvalue::Repeat(..) |

compiler/rustc_middle/src/mir/mod.rs

@@ -2008,7 +2008,9 @@ pub enum Rvalue<'tcx> {
     /// Accessing a thread local static. This is inherently a runtime operation, even if llvm
     /// treats it as an access to a static. This `Rvalue` yields a reference to the thread local
     /// static.
-    ThreadLocalRef(DefId),
+    /// The `Ty` field is the type at which the access is happening, so e.g. a
+    /// `&raw SOME_TLS` will yield a raw pointer, while `&SOME_TLS` will yield a reference.
+    ThreadLocalRef(DefId, Ty<'tcx>),
 
     /// Create a raw pointer to the given place
     /// Can be generated by raw address of expressions (`&raw const x`),
@@ -2149,9 +2151,14 @@ impl<'tcx> Debug for Rvalue<'tcx> {
             UnaryOp(ref op, ref a) => write!(fmt, "{:?}({:?})", op, a),
             Discriminant(ref place) => write!(fmt, "discriminant({:?})", place),
             NullaryOp(ref op, ref t) => write!(fmt, "{:?}({:?})", op, t),
-            ThreadLocalRef(did) => ty::tls::with(|tcx| {
-                let muta = tcx.static_mutability(did).unwrap().prefix_str();
-                write!(fmt, "&/*tls*/ {}{}", muta, tcx.def_path_str(did))
+            ThreadLocalRef(did, ty) => ty::tls::with(|tcx| {
+                let muta = ty.builtin_deref(true).unwrap().mutbl.prefix_str();
+                let raw = match ty.kind() {
+                    ty::Ref(..) => "",
+                    ty::RawPtr(..) => "raw ",
+                    _ => bug!("Rvalue::ThreadLocalRef types can only be pointer or reference"),
+                };
+                write!(fmt, "&{}/*tls*/ {}{}", raw, muta, tcx.def_path_str(did))
             }),
             Ref(region, borrow_kind, ref place) => {
                 let kind_str = match borrow_kind {

compiler/rustc_middle/src/mir/tcx.rs

@@ -151,13 +151,7 @@ impl<'tcx> Rvalue<'tcx> {
             Rvalue::Repeat(ref operand, count) => {
                 tcx.mk_ty(ty::Array(operand.ty(local_decls, tcx), count))
             }
-            Rvalue::ThreadLocalRef(did) => {
-                if tcx.is_mutable_static(did) {
-                    tcx.mk_mut_ptr(tcx.type_of(did))
-                } else {
-                    tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.type_of(did))
-                }
-            }
+            Rvalue::ThreadLocalRef(_, ty) => ty,
             Rvalue::Ref(reg, bk, ref place) => {
                 let place_ty = place.ty(local_decls, tcx).ty;
                 tcx.mk_ref(reg, ty::TypeAndMut { ty: place_ty, mutbl: bk.to_mutbl_lossy() })

compiler/rustc_middle/src/mir/type_foldable.rs

@@ -176,7 +176,7 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
         match *self {
             Use(ref op) => Use(op.fold_with(folder)),
             Repeat(ref op, len) => Repeat(op.fold_with(folder), len.fold_with(folder)),
-            ThreadLocalRef(did) => ThreadLocalRef(did.fold_with(folder)),
+            ThreadLocalRef(did, ty) => ThreadLocalRef(did.fold_with(folder), ty.fold_with(folder)),
             Ref(region, bk, ref place) => {
                 Ref(region.fold_with(folder), bk, place.fold_with(folder))
             }
@@ -220,7 +220,7 @@ impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
         match *self {
             Use(ref op) => op.visit_with(visitor),
             Repeat(ref op, _) => op.visit_with(visitor),
-            ThreadLocalRef(did) => did.visit_with(visitor),
+            ThreadLocalRef(did, ty) => did.visit_with(visitor) || ty.visit_with(visitor),
             Ref(region, _, ref place) => region.visit_with(visitor) || place.visit_with(visitor),
             AddressOf(_, ref place) => place.visit_with(visitor),
             Len(ref place) => place.visit_with(visitor),

compiler/rustc_middle/src/mir/visit.rs

@@ -608,7 +608,7 @@ macro_rules! make_mir_visitor {
                         self.visit_operand(value, location);
                     }
 
-                    Rvalue::ThreadLocalRef(_) => {}
+                    Rvalue::ThreadLocalRef(_, ty) => self.visit_ty(ty, TyContext::Location(location)),
 
                     Rvalue::Ref(r, bk, path) => {
                         self.visit_region(r, location);

compiler/rustc_mir/src/borrow_check/invalidation.rs

@@ -303,7 +303,7 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
                 self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
             }
 
-            Rvalue::ThreadLocalRef(_) => {}
+            Rvalue::ThreadLocalRef(..) => {}
 
             Rvalue::Use(ref operand)
             | Rvalue::Repeat(ref operand, _)

compiler/rustc_mir/src/borrow_check/mod.rs

@@ -1319,7 +1319,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 );
             }
 
-            Rvalue::ThreadLocalRef(_) => {}
+            Rvalue::ThreadLocalRef(..) => {}
 
             Rvalue::Use(ref operand)
             | Rvalue::Repeat(ref operand, _)

compiler/rustc_mir/src/borrow_check/type_check/mod.rs

@@ -2369,7 +2369,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     fn rvalue_user_ty(&self, rvalue: &Rvalue<'tcx>) -> Option<UserTypeAnnotationIndex> {
         match rvalue {
             Rvalue::Use(_)
-            | Rvalue::ThreadLocalRef(_)
+            | Rvalue::ThreadLocalRef(..)
             | Rvalue::Repeat(..)
             | Rvalue::Ref(..)
             | Rvalue::AddressOf(..)

compiler/rustc_mir/src/dataflow/move_paths/builder.rs

@@ -324,7 +324,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
 
     fn gather_rvalue(&mut self, rvalue: &Rvalue<'tcx>) {
         match *rvalue {
-            Rvalue::ThreadLocalRef(_) => {} // not-a-move
+            Rvalue::ThreadLocalRef(..) => {} // not-a-move
             Rvalue::Use(ref operand)
             | Rvalue::Repeat(ref operand, _)
             | Rvalue::Cast(_, ref operand, _)

compiler/rustc_mir/src/interpret/step.rs

@@ -155,7 +155,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         use rustc_middle::mir::Rvalue::*;
         match *rvalue {
-            ThreadLocalRef(did) => {
+            ThreadLocalRef(did, _) => {
                 let id = M::thread_local_static_alloc_id(self, did)?;
                 let val = self.global_base_pointer(id.into())?;
                 self.write_scalar(val, dest)?;

compiler/rustc_mir/src/monomorphize/collector.rs

@@ -610,7 +610,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
                     self.output.push(create_fn_mono_item(self.tcx, instance, span));
                 }
             }
-            mir::Rvalue::ThreadLocalRef(def_id) => {
+            mir::Rvalue::ThreadLocalRef(def_id, _) => {
                 assert!(self.tcx.is_thread_local_static(def_id));
                 let instance = Instance::mono(self.tcx, def_id);
                 if should_codegen_locally(self.tcx, &instance) {

compiler/rustc_mir/src/transform/check_consts/qualifs.rs

@@ -149,7 +149,7 @@ where
     F: FnMut(Local) -> bool,
 {
     match rvalue {
-        Rvalue::ThreadLocalRef(_) | Rvalue::NullaryOp(..) => {
+        Rvalue::ThreadLocalRef(..) | Rvalue::NullaryOp(..) => {
             Q::in_any_value_of_ty(cx, rvalue.ty(cx.body, cx.tcx))
         }
 

compiler/rustc_mir/src/transform/check_consts/validation.rs

@@ -305,7 +305,7 @@ impl Visitor<'tcx> for Validator<'mir, 'tcx> {
         self.super_rvalue(rvalue, location);
 
         match *rvalue {
-            Rvalue::ThreadLocalRef(_) => self.check_op(ops::ThreadLocalAccess),
+            Rvalue::ThreadLocalRef(..) => self.check_op(ops::ThreadLocalAccess),
 
             Rvalue::Use(_)
             | Rvalue::Repeat(..)

compiler/rustc_mir/src/transform/const_prop.rs

@@ -684,8 +684,8 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
 
                 return None;
             }
-            Rvalue::ThreadLocalRef(def_id) => {
-                trace!("skipping ThreadLocalRef({:?})", def_id);
+            Rvalue::ThreadLocalRef(def_id, ty) => {
+                trace!("skipping ThreadLocalRef({:?}, {})", def_id, ty);
 
                 return None;
             }

compiler/rustc_mir/src/transform/promote_consts.rs

@@ -645,7 +645,7 @@ impl<'tcx> Validator<'_, 'tcx> {
         }
 
         match rvalue {
-            Rvalue::ThreadLocalRef(_) => Err(Unpromotable),
+            Rvalue::ThreadLocalRef(..) => Err(Unpromotable),
 
             Rvalue::NullaryOp(..) => Ok(()),
 

compiler/rustc_mir/src/transform/qualify_min_const_fn.rs

@@ -158,7 +158,7 @@ fn check_rvalue(
     span: Span,
 ) -> McfResult {
     match rvalue {
-        Rvalue::ThreadLocalRef(_) => {
+        Rvalue::ThreadLocalRef(..) => {
             Err((span, "cannot access thread local storage in const fn".into()))
         }
         Rvalue::Repeat(operand, _) | Rvalue::Use(operand) => {

compiler/rustc_mir_build/src/build/expr/as_place.rs

@@ -258,7 +258,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::InlineAsm { .. }
             | ExprKind::LlvmInlineAsm { .. }
             | ExprKind::Yield { .. }
-            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::ThreadLocalRef(..)
             | ExprKind::Call { .. } => {
                 // these are not places, so we need to make a temporary.
                 debug_assert!(match Category::of(&expr.kind) {

compiler/rustc_mir_build/src/build/expr/as_rvalue.rs

@@ -53,7 +53,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let source_info = this.source_info(expr_span);
 
         match expr.kind {
-            ExprKind::ThreadLocalRef(did) => block.and(Rvalue::ThreadLocalRef(did)),
+            ExprKind::ThreadLocalRef(did, ty) => block.and(Rvalue::ThreadLocalRef(did, ty)),
             ExprKind::Scope { region_scope, lint_level, value } => {
                 let region_scope = (region_scope, source_info);
                 this.in_scope(region_scope, lint_level, |this| this.as_rvalue(block, scope, value))

compiler/rustc_mir_build/src/build/expr/as_temp.rs

@@ -70,7 +70,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     local_decl.local_info =
                         Some(box LocalInfo::StaticRef { def_id, is_thread_local: false });
                 }
-                ExprKind::ThreadLocalRef(def_id) => {
+                ExprKind::ThreadLocalRef(def_id, _) => {
                     assert!(this.hir.tcx().is_thread_local_static(def_id));
                     local_decl.internal = true;
                     local_decl.local_info =

compiler/rustc_mir_build/src/build/expr/category.rs

@@ -65,7 +65,7 @@ impl Category {
             | ExprKind::Repeat { .. }
             | ExprKind::Assign { .. }
             | ExprKind::AssignOp { .. }
-            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::ThreadLocalRef(..)
             | ExprKind::LlvmInlineAsm { .. } => Some(Category::Rvalue(RvalueFunc::AsRvalue)),
 
             ExprKind::Literal { .. } | ExprKind::StaticRef { .. } => Some(Category::Constant),

compiler/rustc_mir_build/src/build/expr/into.rs

@@ -457,7 +457,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::Tuple { .. }
             | ExprKind::Closure { .. }
             | ExprKind::Literal { .. }
-            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::ThreadLocalRef(..)
             | ExprKind::StaticRef { .. } => {
                 debug_assert!(match Category::of(&expr.kind).unwrap() {
                     // should be handled above

compiler/rustc_mir_build/src/thir/cx/expr.rs

@@ -229,12 +229,8 @@ fn make_mirror_unadjusted<'a, 'tcx>(
             }
         }
 
-        hir::ExprKind::AddrOf(hir::BorrowKind::Ref, mutbl, ref arg) => {
-            ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg: arg.to_ref() }
-        }
-
-        hir::ExprKind::AddrOf(hir::BorrowKind::Raw, mutability, ref arg) => {
-            ExprKind::AddressOf { mutability, arg: arg.to_ref() }
+        hir::ExprKind::AddrOf(kind, mutability, arg) => {
+            convert_addr_of_expr(cx, kind, mutability, arg)
         }
 
         hir::ExprKind::Block(ref blk, _) => ExprKind::Block { body: &blk },
@@ -863,15 +859,7 @@ fn convert_path_expr<'a, 'tcx>(
         Res::Def(DefKind::Static, id) => {
             let ty = cx.tcx.static_ptr_ty(id);
             let temp_lifetime = cx.region_scope_tree.temporary_scope(expr.hir_id.local_id);
-            let kind = if cx.tcx.is_thread_local_static(id) {
-                ExprKind::ThreadLocalRef(id)
-            } else {
-                let ptr = cx.tcx.create_static_alloc(id);
-                ExprKind::StaticRef {
-                    literal: ty::Const::from_scalar(cx.tcx, Scalar::Ptr(ptr.into()), ty),
-                    def_id: id,
-                }
-            };
+            let kind = convert_static_ref(cx, id, ty);
             ExprKind::Deref { arg: Expr { ty, temp_lifetime, span: expr.span, kind }.to_ref() }
         }
 
@@ -881,6 +869,47 @@ fn convert_path_expr<'a, 'tcx>(
     }
 }
 
+fn convert_static_ref<'tcx>(cx: &mut Cx<'_, 'tcx>, id: DefId, ty: Ty<'tcx>) -> ExprKind<'tcx> {
+    if cx.tcx.is_thread_local_static(id) {
+        ExprKind::ThreadLocalRef(id, ty)
+    } else {
+        let ptr = cx.tcx.create_static_alloc(id);
+        ExprKind::StaticRef {
+            literal: ty::Const::from_scalar(cx.tcx, Scalar::Ptr(ptr.into()), ty),
+            def_id: id,
+        }
+    }
+}
+
+fn convert_addr_of_expr<'tcx>(
+    cx: &mut Cx<'_, 'tcx>,
+    kind: hir::BorrowKind,
+    mutability: hir::Mutability,
+    arg: &'tcx hir::Expr<'tcx>,
+) -> ExprKind<'tcx> {
+    // Fast path so that taking a reference to a static doesn't end up
+    // as `&*&STATIC` but just `&STATIC`
+    if let hir::ExprKind::Path(qpath) = &arg.kind {
+        let res = cx.typeck_results().qpath_res(qpath, arg.hir_id);
+        if let Res::Def(DefKind::Static, id) = res {
+            let mut tm = cx.tcx.static_ptr_ty(id).builtin_deref(true).unwrap();
+            tm.mutbl = mutability;
+            let ty = match kind {
+                hir::BorrowKind::Ref => cx.tcx.mk_ref(cx.tcx.lifetimes.re_erased, tm),
+                hir::BorrowKind::Raw => cx.tcx.mk_ptr(tm),
+            };
+            return convert_static_ref(cx, id, ty);
+        }
+    }
+    match kind {
+        hir::BorrowKind::Ref => {
+            ExprKind::Borrow { borrow_kind: mutability.to_borrow_kind(), arg: arg.to_ref() }
+        }
+
+        hir::BorrowKind::Raw => ExprKind::AddressOf { mutability, arg: arg.to_ref() },
+    }
+}
+
 fn convert_var<'tcx>(
     cx: &mut Cx<'_, 'tcx>,
     expr: &'tcx hir::Expr<'tcx>,

compiler/rustc_mir_build/src/thir/mod.rs

@@ -293,7 +293,7 @@ crate enum ExprKind<'tcx> {
         line_spans: &'tcx [Span],
     },
     /// An expression taking a reference to a thread local.
-    ThreadLocalRef(DefId),
+    ThreadLocalRef(DefId, Ty<'tcx>),
     LlvmInlineAsm {
         asm: &'tcx hir::LlvmInlineAsmInner,
         outputs: Vec<ExprRef<'tcx>>,

src/test/mir-opt/const_promotion_extern_static.BAR-promoted[0].ConstProp.after.mir

@@ -3,19 +3,15 @@
 promoted[0] in BAR: &[&i32; 1] = {
     let mut _0: &[&i32; 1];              // return place in scope 0 at $DIR/const-promotion-extern-static.rs:9:31: 9:35
     let mut _1: [&i32; 1];               // in scope 0 at $DIR/const-promotion-extern-static.rs:9:31: 9:35
-    let mut _2: &i32;                    // in scope 0 at $DIR/const-promotion-extern-static.rs:9:32: 9:34
-    let mut _3: &i32;                    // in scope 0 at $DIR/const-promotion-extern-static.rs:9:33: 9:34
 
     bb0: {
-        _3 = const {alloc0: &i32};       // scope 0 at $DIR/const-promotion-extern-static.rs:9:33: 9:34
+        _1 = [const {alloc0: &i32}];     // scope 0 at $DIR/const-promotion-extern-static.rs:9:31: 9:35
                                          // ty::Const
                                          // + ty: &i32
                                          // + val: Value(Scalar(alloc0))
                                          // mir::Constant
-                                         // + span: $DIR/const-promotion-extern-static.rs:9:33: 9:34
+                                         // + span: $DIR/const-promotion-extern-static.rs:9:32: 9:34
                                          // + literal: Const { ty: &i32, val: Value(Scalar(alloc0)) }
-        _2 = _3;                         // scope 0 at $DIR/const-promotion-extern-static.rs:9:32: 9:34
-        _1 = [move _2];                  // scope 0 at $DIR/const-promotion-extern-static.rs:9:31: 9:35
         _0 = &_1;                        // scope 0 at $DIR/const-promotion-extern-static.rs:9:31: 9:35
         return;                          // scope 0 at $DIR/const-promotion-extern-static.rs:9:31: 9:35
     }