From 7cc2dd81f3d522355dd67788c9fb0976f1da88c2 Mon Sep 17 00:00:00 2001 From: Ding Xiang Fei Date: Tue, 22 Apr 2025 19:28:51 +0000 Subject: [PATCH 1/2] relocate upvars with saved locals for analysis ... and treat coroutine upvar captures as saved locals as well. This allows the liveness analysis to determine which captures are truly saved across a yield point and which are initially used but discarded at first yield points. In the event that upvar captures are promoted, most certainly because a coroutine suspends at least once, the slots in the promotion prefix shall be reused. This means that the copies emitted in the upvar relocation MIR pass will eventually elided and eliminated in the codegen phase, hence no additional runtime cost is realised. Additional MIR dumps are inserted so that it is easier to inspect the bodies of async closures, including those that captures the state by-value. Debug information is updated to point at the correct location for upvars in borrow checking errors and final debuginfo. A language change that this patch enables is now actually reverted, so that lifetimes on relocated upvars are invariant with the upvars outside of the coroutine body. We are deferring the language change to a later discussion. Co-authored-by: Dario Nieuwenhuis Signed-off-by: Xiangfei Ding --- compiler/rustc_abi/src/layout.rs | 9 +- compiler/rustc_abi/src/layout/coroutine.rs | 132 ++++- compiler/rustc_abi/src/lib.rs | 4 +- .../src/diagnostics/mutability_errors.rs | 96 +-- compiler/rustc_borrowck/src/lib.rs | 2 +- compiler/rustc_borrowck/src/path_utils.rs | 4 +- compiler/rustc_borrowck/src/type_check/mod.rs | 21 +- compiler/rustc_codegen_cranelift/src/base.rs | 3 + .../src/debuginfo/metadata.rs | 6 +- .../src/debuginfo/metadata/enums/cpp_like.rs | 2 - .../src/debuginfo/metadata/enums/mod.rs | 35 +- .../src/debuginfo/metadata/enums/native.rs | 4 - compiler/rustc_codegen_ssa/src/mir/operand.rs | 9 +- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 3 + .../rustc_const_eval/src/interpret/step.rs | 3 + compiler/rustc_index/src/vec.rs | 35 ++ compiler/rustc_middle/src/mir/query.rs | 20 + compiler/rustc_middle/src/mir/statement.rs | 14 +- compiler/rustc_middle/src/ty/instance.rs | 7 +- compiler/rustc_middle/src/ty/layout.rs | 5 +- compiler/rustc_middle/src/ty/mod.rs | 84 ++- compiler/rustc_middle/src/ty/sty.rs | 7 - compiler/rustc_mir_transform/src/coroutine.rs | 278 +++++++-- .../src/coroutine/by_move_body.rs | 64 +- .../rustc_mir_transform/src/coroutine/drop.rs | 18 +- .../src/coroutine/relocate_upvars.rs | 557 ++++++++++++++++++ .../src/deref_separator.rs | 1 + compiler/rustc_mir_transform/src/lib.rs | 2 +- .../rustc_mir_transform/src/pass_manager.rs | 13 +- compiler/rustc_mir_transform/src/patch.rs | 4 + compiler/rustc_mir_transform/src/shim.rs | 6 +- .../src/shim/async_destructor_ctor.rs | 62 +- compiler/rustc_mir_transform/src/simplify.rs | 8 +- compiler/rustc_mir_transform/src/validate.rs | 13 +- compiler/rustc_session/src/config.rs | 31 +- compiler/rustc_session/src/options.rs | 15 + .../src/error_reporting/traits/suggestions.rs | 24 +- .../src/traits/query/dropck_outlives.rs | 11 +- .../rustc_traits/src/coroutine_witnesses.rs | 26 +- compiler/rustc_ty_utils/src/layout.rs | 32 +- .../clippy/tests/ui/future_not_send.stderr | 4 +- .../clippy/tests/ui/large_futures.stderr | 10 +- .../clippy/tests/ui/needless_lifetimes.fixed | 1 + .../clippy/tests/ui/needless_lifetimes.rs | 1 + .../clippy/tests/ui/needless_lifetimes.stderr | 84 +-- src/tools/miri/tests/pass/async-drop.rs | 8 +- .../pass/async-drop.stackrelocate.stdout | 23 + .../tests/pass/async-drop.treerelocate.stdout | 23 + tests/codegen-llvm/async-fn-debug.rs | 10 +- tests/debuginfo/coroutine-objects.rs | 12 +- .../debuginfo/coroutine-objects_relocated.rs | 89 +++ ...#0}.coroutine_drop_async.0.panic-abort.mir | 4 +- ...0}.coroutine_drop_async.0.panic-unwind.mir | 4 +- ...await.a-{closure#0}.coroutine_resume.0.mir | 1 + ...await.b-{closure#0}.coroutine_resume.0.mir | 41 +- ....main-{closure#0}.StateTransform.after.mir | 1 + ....main-{closure#1}.StateTransform.after.mir | 1 + ...re#0}.RelocateUpvars.after.panic-abort.mir | 115 ++++ ...e#0}.RelocateUpvars.after.panic-unwind.mir | 139 +++++ ...e#0}.RelocateUpvars.before.panic-abort.mir | 63 ++ ...#0}.RelocateUpvars.before.panic-unwind.mir | 78 +++ tests/mir-opt/coroutine_relocate_upvars.rs | 16 + ...ny.main-{closure#0}.coroutine_resume.0.mir | 1 + ...y.run2-{closure#0}.Inline.panic-abort.diff | 2 +- ....run2-{closure#0}.Inline.panic-unwind.diff | 2 +- ... => async-drop-initial.classic.run.stdout} | 1 + .../async-drop-initial.relocate.run.stdout | 23 + .../async-drop/async-drop-initial.rs | 81 +-- .../async-awaiting-fut.classic.stdout | 113 ++++ ...out => async-awaiting-fut.relocate.stdout} | 54 +- .../future-sizes/async-awaiting-fut.rs | 3 + .../async-await/future-sizes/future-as-arg.rs | 9 +- .../future-sizes/large-arg.classic.stdout | 106 ++++ ...e-arg.stdout => large-arg.relocate.stdout} | 71 +-- .../ui/async-await/future-sizes/large-arg.rs | 3 + tests/ui/async-await/issue-70818.stderr | 12 +- .../issue-74072-lifetime-name-annotations.rs | 6 +- ...sue-74072-lifetime-name-annotations.stderr | 4 +- tests/ui/async-await/issue-86507.rs | 7 +- tests/ui/async-await/issue-86507.stderr | 14 +- .../cmse-nonsecure-entry/c-variadic.rs | 15 +- .../cmse-nonsecure-entry/c-variadic.stderr | 6 +- ...-impl.stderr => clone-impl.classic.stderr} | 76 +-- tests/ui/coroutine/clone-impl.relocate.stderr | 115 ++++ tests/ui/coroutine/clone-impl.rs | 5 +- .../delayed-obligations-emit.next.stderr | 1 + .../ref-escapes-but-not-over-yield.rs | 7 +- .../ref-escapes-but-not-over-yield.stderr | 2 +- tests/ui/coroutine/ref-upvar-not-send.rs | 30 +- tests/ui/coroutine/ref-upvar-not-send.stderr | 40 +- .../coroutine/unsized-capture-across-yield.rs | 3 +- .../large_future.attribute.stderr | 8 +- .../large_future.option-relocate.stderr | 23 + .../large_future.option.stderr | 8 +- .../ui/lint/large_assignments/large_future.rs | 6 +- tests/ui/mir/lint/storage-live.stderr | 2 +- .../{async.stdout => async.classic.stdout} | 9 +- .../ui/print_type_sizes/async.relocate.stdout | 65 ++ tests/ui/print_type_sizes/async.rs | 3 + .../print_type_sizes/coroutine.classic.stdout | 15 + .../coroutine.relocate.stdout | 15 + tests/ui/print_type_sizes/coroutine.rs | 3 + tests/ui/print_type_sizes/coroutine.stdout | 10 - ...re.stdout => async-closure.classic.stdout} | 46 +- .../async-closure.relocate.stdout | 127 ++++ .../ui/rustc_public-ir-print/async-closure.rs | 3 + 106 files changed, 2891 insertions(+), 662 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs create mode 100644 src/tools/miri/tests/pass/async-drop.stackrelocate.stdout create mode 100644 src/tools/miri/tests/pass/async-drop.treerelocate.stdout create mode 100644 tests/debuginfo/coroutine-objects_relocated.rs create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir create mode 100644 tests/mir-opt/coroutine_relocate_upvars.rs rename tests/ui/async-await/async-drop/{async-drop-initial.run.stdout => async-drop-initial.classic.run.stdout} (94%) create mode 100644 tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout create mode 100644 tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout rename tests/ui/async-await/future-sizes/{async-awaiting-fut.stdout => async-awaiting-fut.relocate.stdout} (77%) create mode 100644 tests/ui/async-await/future-sizes/large-arg.classic.stdout rename tests/ui/async-await/future-sizes/{large-arg.stdout => large-arg.relocate.stdout} (68%) rename tests/ui/coroutine/{clone-impl.stderr => clone-impl.classic.stderr} (58%) create mode 100644 tests/ui/coroutine/clone-impl.relocate.stderr create mode 100644 tests/ui/lint/large_assignments/large_future.option-relocate.stderr rename tests/ui/print_type_sizes/{async.stdout => async.classic.stdout} (92%) create mode 100644 tests/ui/print_type_sizes/async.relocate.stdout create mode 100644 tests/ui/print_type_sizes/coroutine.classic.stdout create mode 100644 tests/ui/print_type_sizes/coroutine.relocate.stdout delete mode 100644 tests/ui/print_type_sizes/coroutine.stdout rename tests/ui/rustc_public-ir-print/{async-closure.stdout => async-closure.classic.stdout} (61%) create mode 100644 tests/ui/rustc_public-ir-print/async-closure.relocate.stdout diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 14356813b7bb0..d1668fd50f445 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -3,6 +3,7 @@ use std::fmt::{self, Write}; use std::ops::{Bound, Deref}; use std::{cmp, iter}; +pub use coroutine::PackCoroutineLayout; use rustc_hashes::Hash64; use rustc_index::Idx; use rustc_index::bit_set::BitMatrix; @@ -210,17 +211,21 @@ impl LayoutCalculator { >( &self, local_layouts: &IndexSlice, - prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> LayoutCalculatorResult { coroutine::layout( self, local_layouts, - prefix_layouts, + relocated_upvars, + upvar_layouts, variant_fields, storage_conflicts, + pack, tag_to_layout, ) } diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs index 2b22276d4aed7..bbe2a2e6a61db 100644 --- a/compiler/rustc_abi/src/layout/coroutine.rs +++ b/compiler/rustc_abi/src/layout/coroutine.rs @@ -30,6 +30,17 @@ use crate::{ StructKind, TagEncoding, Variants, WrappingRange, }; +/// This option controls how coroutine saved locals are packed +/// into the coroutine state data +#[derive(Debug, Clone, Copy)] +pub enum PackCoroutineLayout { + /// The classic layout where captures are always promoted to coroutine state prefix + Classic, + /// Captures are first saved into the `UNRESUME` state and promoted + /// when they are used across more than one suspension + CapturesOnly, +} + /// Overlap eligibility and variant assignment for each CoroutineSavedLocal. #[derive(Clone, Debug, PartialEq)] enum SavedLocalEligibility { @@ -145,9 +156,11 @@ pub(super) fn layout< >( calc: &super::LayoutCalculator, local_layouts: &IndexSlice, - mut prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> super::LayoutCalculatorResult { use SavedLocalEligibility::*; @@ -155,10 +168,11 @@ pub(super) fn layout< let (ineligible_locals, assignments) = coroutine_saved_local_eligibility(local_layouts.len(), variant_fields, storage_conflicts); - // Build a prefix layout, including "promoting" all ineligible - // locals as part of the prefix. We compute the layout of all of - // these fields at once to get optimal packing. - let tag_index = prefix_layouts.next_index(); + // Build a prefix layout, consisting of only the state tag and, as per request, upvars + let tag_index = match pack { + PackCoroutineLayout::CapturesOnly => FieldIdx::new(0), + PackCoroutineLayout::Classic => upvar_layouts.next_index(), + }; // `variant_fields` already accounts for the reserved variants, so no need to add them. let max_discr = (variant_fields.len() - 1) as u128; @@ -169,17 +183,28 @@ pub(super) fn layout< }; let promoted_layouts = ineligible_locals.iter().map(|local| local_layouts[local]); - prefix_layouts.push(tag_to_layout(tag)); - prefix_layouts.extend(promoted_layouts); + // FIXME: when we introduce more pack scheme, we need to change the prefix layout here + let prefix_layouts: IndexVec<_, _> = match pack { + PackCoroutineLayout::Classic => { + // Classic scheme packs the states as follows + // [ .. , , ] ++ + // In addition, UNRESUME overlaps with the part + upvar_layouts.into_iter().chain([tag_to_layout(tag)]).chain(promoted_layouts).collect() + } + PackCoroutineLayout::CapturesOnly => { + [tag_to_layout(tag)].into_iter().chain(promoted_layouts).collect() + } + }; + debug!(?prefix_layouts, ?pack); let prefix = calc.univariant(&prefix_layouts, &ReprOptions::default(), StructKind::AlwaysSized)?; let (prefix_size, prefix_align) = (prefix.size, prefix.align); - // Split the prefix layout into the "outer" fields (upvars and - // discriminant) and the "promoted" fields. Promoted fields will - // get included in each variant that requested them in - // CoroutineLayout. + // Split the prefix layout into the discriminant and + // the "promoted" fields. + // Promoted fields will get included in each variant + // that requested them in CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { FieldsShape::Arbitrary { mut offsets, memory_index } => { @@ -218,19 +243,67 @@ pub(super) fn layout< let variants = variant_fields .iter_enumerated() .map(|(index, variant_fields)| { + let is_unresumed = index == VariantIdx::new(0); + if is_unresumed && matches!(pack, PackCoroutineLayout::Classic) { + let fields = FieldsShape::Arbitrary { + offsets: (0..tag_index.index()).map(|i| outer_fields.offset(i)).collect(), + memory_index: (0..tag_index.index()) + .map(|i| { + (outer_fields.memory_index(i) + promoted_memory_index.len()) as u32 + }) + .collect(), + }; + let align = prefix.align; + let size = prefix.size; + return Ok(LayoutData { + fields, + variants: Variants::Single { index }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + uninhabited: false, + align, + size, + max_repr_align: None, + unadjusted_abi_align: align.abi, + randomization_seed: Default::default(), + }); + } + let mut is_ineligible = IndexVec::from_elem_n(None, variant_fields.len()); + for (field, &local) in variant_fields.iter_enumerated() { + if is_unresumed { + if let Some(inner_local) = relocated_upvars[local] + && let Ineligible(Some(promoted_field)) = assignments[inner_local] + { + is_ineligible.insert(field, promoted_field); + continue; + } + } + match assignments[local] { + Assigned(v) if v == index => {} + Ineligible(Some(promoted_field)) => { + is_ineligible.insert(field, promoted_field); + } + Ineligible(None) => { + panic!("an ineligible local should have been promoted into the prefix") + } + Assigned(_) => { + panic!("an eligible local should have been assigned to exactly one variant") + } + Unassigned => { + panic!("each saved local should have been inspected at least once") + } + } + } // Only include overlap-eligible fields when we compute our variant layout. - let variant_only_tys = variant_fields - .iter() - .filter(|local| match assignments[**local] { - Unassigned => unreachable!(), - Assigned(v) if v == index => true, - Assigned(_) => unreachable!("assignment does not match variant"), - Ineligible(_) => false, + let fields: IndexVec<_, _> = variant_fields + .iter_enumerated() + .filter_map(|(field, &local)| { + if is_ineligible.contains(field) { None } else { Some(local_layouts[local]) } }) - .map(|local| local_layouts[*local]); + .collect(); let mut variant = calc.univariant( - &variant_only_tys.collect::>(), + &fields, &ReprOptions::default(), StructKind::Prefixed(prefix_size, prefix_align.abi), )?; @@ -254,19 +327,14 @@ pub(super) fn layout< IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx); let mut offsets_and_memory_index = iter::zip(offsets, memory_index); - let combined_offsets = variant_fields + let combined_offsets = is_ineligible .iter_enumerated() - .map(|(i, local)| { - let (offset, memory_index) = match assignments[*local] { - Unassigned => unreachable!(), - Assigned(_) => { - let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); - (offset, promoted_memory_index.len() as u32 + memory_index) - } - Ineligible(field_idx) => { - let field_idx = field_idx.unwrap(); - (promoted_offsets[field_idx], promoted_memory_index[field_idx]) - } + .map(|(i, &is_ineligible)| { + let (offset, memory_index) = if let Some(field_idx) = is_ineligible { + (promoted_offsets[field_idx], promoted_memory_index[field_idx]) + } else { + let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); + (offset, promoted_memory_index.len() as u32 + memory_index) }; combined_inverse_memory_index[memory_index] = i; offset diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index de44c8755a078..fa9d947cd9f1d 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -67,7 +67,9 @@ pub use canon_abi::{ArmCall, CanonAbi, InterruptKind, X86Call}; pub use extern_abi::CVariadicStatus; pub use extern_abi::{ExternAbi, all_names}; #[cfg(feature = "nightly")] -pub use layout::{FIRST_VARIANT, FieldIdx, Layout, TyAbiInterface, TyAndLayout, VariantIdx}; +pub use layout::{ + FIRST_VARIANT, FieldIdx, Layout, PackCoroutineLayout, TyAbiInterface, TyAndLayout, VariantIdx, +}; pub use layout::{LayoutCalculator, LayoutCalculatorError}; /// Requirements for a `StableHashingContext` to be used in this crate. diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs index 727cf19cd8bb3..9cd469e518cbb 100644 --- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs @@ -421,49 +421,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { Place::ty_from(local, proj_base, self.body, self.infcx.tcx).ty )); - let captured_place = self.upvars[upvar_index.index()]; - - err.span_label(span, format!("cannot {act}")); - - let upvar_hir_id = captured_place.get_root_variable(); - - if let Node::Pat(pat) = self.infcx.tcx.hir_node(upvar_hir_id) - && let hir::PatKind::Binding(hir::BindingMode::NONE, _, upvar_ident, _) = - pat.kind - { - if upvar_ident.name == kw::SelfLower { - for (_, node) in self.infcx.tcx.hir_parent_iter(upvar_hir_id) { - if let Some(fn_decl) = node.fn_decl() { - if !matches!( - fn_decl.implicit_self, - hir::ImplicitSelfKind::RefImm | hir::ImplicitSelfKind::RefMut - ) { - err.span_suggestion_verbose( - upvar_ident.span.shrink_to_lo(), - "consider changing this to be mutable", - "mut ", - Applicability::MachineApplicable, - ); - break; - } - } - } - } else { - err.span_suggestion_verbose( - upvar_ident.span.shrink_to_lo(), - "consider changing this to be mutable", - "mut ", - Applicability::MachineApplicable, - ); - } - } - - let tcx = self.infcx.tcx; - if let ty::Ref(_, ty, Mutability::Mut) = the_place_err.ty(self.body, tcx).ty.kind() - && let ty::Closure(id, _) = *ty.kind() - { - self.show_mutating_upvar(tcx, id.expect_local(), the_place_err, &mut err); - } + self.suggest_mutable_upvar(*upvar_index, the_place_err, &mut err, span, act); } // Complete hack to approximate old AST-borrowck diagnostic: if the span starts @@ -569,6 +527,58 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } } + fn suggest_mutable_upvar( + &self, + upvar_index: FieldIdx, + the_place_err: PlaceRef<'tcx>, + err: &mut Diag<'infcx>, + span: Span, + act: &str, + ) { + let captured_place = self.upvars[upvar_index.index()]; + + err.span_label(span, format!("cannot {act}")); + + let upvar_hir_id = captured_place.get_root_variable(); + + if let Node::Pat(pat) = self.infcx.tcx.hir_node(upvar_hir_id) + && let hir::PatKind::Binding(hir::BindingMode::NONE, _, upvar_ident, _) = pat.kind + { + if upvar_ident.name == kw::SelfLower { + for (_, node) in self.infcx.tcx.hir_parent_iter(upvar_hir_id) { + if let Some(fn_decl) = node.fn_decl() { + if !matches!( + fn_decl.implicit_self, + hir::ImplicitSelfKind::RefImm | hir::ImplicitSelfKind::RefMut + ) { + err.span_suggestion_verbose( + upvar_ident.span.shrink_to_lo(), + "consider changing this to be mutable", + "mut ", + Applicability::MachineApplicable, + ); + break; + } + } + } + } else { + err.span_suggestion_verbose( + upvar_ident.span.shrink_to_lo(), + "consider changing this to be mutable", + "mut ", + Applicability::MachineApplicable, + ); + } + } + + let tcx = self.infcx.tcx; + if let ty::Ref(_, ty, Mutability::Mut) = the_place_err.ty(self.body, tcx).ty.kind() + && let ty::Closure(id, _) = *ty.kind() + { + self.show_mutating_upvar(tcx, id.expect_local(), the_place_err, err); + } + } + /// Suggest `map[k] = v` => `map.insert(k, v)` and the like. fn suggest_map_index_mut_alternatives(&self, ty: Ty<'tcx>, err: &mut Diag<'infcx>, span: Span) { let Some(adt) = ty.ty_adt_def() else { return }; diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index a85dcf64d8d46..b58a85f6ac6f8 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -2495,7 +2495,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { // If the local may have been initialized, and it is now currently being // mutated, then it is justified to be annotated with the `mut` // keyword, since the mutation may be a possible reassignment. - if is_local_mutation_allowed != LocalMutationIsAllowed::Yes + if !matches!(is_local_mutation_allowed, LocalMutationIsAllowed::Yes) && self.is_local_ever_initialized(local, state).is_some() { self.used_mut.insert(local); diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs index 2c94a32d369ce..6b3169223029a 100644 --- a/compiler/rustc_borrowck/src/path_utils.rs +++ b/compiler/rustc_borrowck/src/path_utils.rs @@ -3,7 +3,7 @@ use std::ops::ControlFlow; use rustc_abi::FieldIdx; use rustc_data_structures::graph::dominators::Dominators; use rustc_middle::mir::{BasicBlock, Body, Location, Place, PlaceRef, ProjectionElem}; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{CapturedPlace, TyCtxt}; use tracing::debug; use crate::borrow_set::{BorrowData, BorrowSet, TwoPhaseActivation}; @@ -133,7 +133,7 @@ pub(super) fn borrow_of_local_data(place: Place<'_>) -> bool { /// of a closure type. pub(crate) fn is_upvar_field_projection<'tcx>( tcx: TyCtxt<'tcx>, - upvars: &[&rustc_middle::ty::CapturedPlace<'tcx>], + upvars: &[&CapturedPlace<'tcx>], place_ref: PlaceRef<'tcx>, body: &Body<'tcx>, ) -> Option { diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 781fb5ba113ab..5de4b3a61bea6 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -25,8 +25,8 @@ use rustc_middle::traits::query::NoSolution; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::{ - self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CoroutineArgsExt, - GenericArgsRef, Ty, TyCtxt, TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, + self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, GenericArgsRef, Ty, TyCtxt, + TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::move_paths::MoveData; @@ -2176,14 +2176,15 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } } AggregateKind::Coroutine(_, args) => { - // It doesn't make sense to look at a field beyond the prefix; - // these require a variant index, and are not initialized in - // aggregate rvalues. - match args.as_coroutine().prefix_tys().get(field_index.as_usize()) { - Some(ty) => Ok(*ty), - None => Err(FieldAccessError::OutOfRange { - field_count: args.as_coroutine().prefix_tys().len(), - }), + // It doesn't make sense to look at a field beyond the captured + // upvars. + // Otherwise it require a variant index, and are not initialized + // in aggregate rvalues. + let upvar_tys = &args.as_coroutine().upvar_tys(); + if let Some(ty) = upvar_tys.get(field_index.as_usize()) { + Ok(*ty) + } else { + Err(FieldAccessError::OutOfRange { field_count: upvar_tys.len() }) } } AggregateKind::CoroutineClosure(_, args) => { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index ebf2ccf74de20..1d4a9915c735a 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -905,6 +905,9 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let variant_dest = lval.downcast_variant(fx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, lval.downcast_variant(fx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, lval, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 014715dd4fda5..7078228262aab 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -15,9 +15,7 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{ HasTypingEnv, LayoutOf, TyAndLayout, WIDE_PTR_ADDR, WIDE_PTR_EXTRA, }; -use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, -}; +use rustc_middle::ty::{self, AdtKind, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility}; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::{ DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Span, Symbol, hygiene, @@ -1152,7 +1150,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { - ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), + ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().upvar_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 4ecc3086e1bdf..769c34f51a04c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -721,7 +721,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( let coroutine_layout = cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.args).unwrap(); - let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); @@ -761,7 +760,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ); let span = coroutine_layout.variant_source_info[variant_index].span; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index caff358607974..81425c71ac645 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -5,12 +5,11 @@ use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_ use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_hir::def::CtorKind; -use rustc_index::IndexSlice; use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; -use rustc_span::{Span, Symbol}; +use rustc_span::Span; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; @@ -287,7 +286,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( coroutine_type_and_layout: TyAndLayout<'tcx>, coroutine_type_di_node: &'ll DIType, coroutine_layout: &CoroutineLayout<'tcx>, - common_upvar_names: &IndexSlice, ) -> &'ll DIType { let variant_name = CoroutineArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( @@ -298,11 +296,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); - let coroutine_args = match coroutine_type_and_layout.ty.kind() { - ty::Coroutine(_, args) => args.as_coroutine(), - _ => unreachable!(), - }; - type_map::build_type_with_children( cx, type_map::stub( @@ -317,7 +310,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( ), |cx, variant_struct_type_di_node| { // Fields that just belong to this variant/state - let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + (0..variant_layout.fields.count()) .map(|field_index| { let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; @@ -340,29 +333,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( None, ) }) - .collect(); - - // Fields that are common to all states - let common_fields: SmallVec<_> = coroutine_args - .prefix_tys() - .iter() - .zip(common_upvar_names) - .enumerate() - .map(|(index, (upvar_ty, upvar_name))| { - build_field_di_node( - cx, - variant_struct_type_di_node, - upvar_name.as_str(), - cx.layout_of(upvar_ty), - coroutine_type_and_layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, upvar_ty), - None, - ) - }) - .collect(); - - state_specific_fields.into_iter().chain(common_fields).collect() + .collect() }, |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 1ae6e6e5eecab..44f4426c699f9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -186,9 +186,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( ) }; - let common_upvar_names = - cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); - // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() @@ -216,7 +213,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ), source_info, } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 88a8e2a844cbc..b187cfcc14b70 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -934,13 +934,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "debug", skip(self, bx), ret)] fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> Option> { - debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); - match self.locals[place_ref.local] { LocalRef::Operand(mut o) => { // We only need to handle the projections that @@ -980,13 +979,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_consume( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_consume(place_ref={:?})", place_ref); - let ty = self.monomorphized_place_ty(place_ref); let layout = bx.cx().layout_of(ty); @@ -1005,13 +1003,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.load_operand(place) } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_operand( &mut self, bx: &mut Bx, operand: &mir::Operand<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_operand(operand={:?})", operand); - match *operand { mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { self.codegen_consume(bx, place.as_ref()) diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index d629003bff5e8..4d367f4675c01 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -167,6 +167,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let variant_dest = dest.project_downcast(bx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_, _) => { + (FIRST_VARIANT, dest.project_downcast(bx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, dest, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 923e00ad4cf1a..ab483a35c43dd 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -309,6 +309,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let variant_dest = self.project_downcast(dest, variant_index)?; (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, self.project_downcast(dest, FIRST_VARIANT)?, None) + } mir::AggregateKind::RawPtr(..) => { // Pointers don't have "fields" in the normal sense, so the // projection-based code below would either fail in projection diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs index 13f0dda180be9..90eb212e69bf5 100644 --- a/compiler/rustc_index/src/vec.rs +++ b/compiler/rustc_index/src/vec.rs @@ -197,6 +197,11 @@ impl IndexVec { pub fn append(&mut self, other: &mut Self) { self.raw.append(&mut other.raw); } + + #[inline] + pub fn debug_map_view(&self) -> IndexSliceMapView<'_, I, T> { + IndexSliceMapView(self.as_slice()) + } } /// `IndexVec` is often used as a map, so it provides some map-like APIs. @@ -220,14 +225,44 @@ impl IndexVec> { pub fn contains(&self, index: I) -> bool { self.get(index).and_then(Option::as_ref).is_some() } + + #[inline] + pub fn debug_map_view_compact(&self) -> IndexSliceMapViewCompact<'_, I, T> { + IndexSliceMapViewCompact(self.as_slice()) + } } +pub struct IndexSliceMapView<'a, I: Idx, T>(&'a IndexSlice); +pub struct IndexSliceMapViewCompact<'a, I: Idx, T>(&'a IndexSlice>); + impl fmt::Debug for IndexVec { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.raw, fmt) } } +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapView<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + entries.entry(&idx, val); + } + entries.finish() + } +} + +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapViewCompact<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + if let Some(val) = val { + entries.entry(&idx, val); + } + } + entries.finish() + } +} + impl Deref for IndexVec { type Target = IndexSlice; diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 2e6c9f207e26b..2d12cf01f4721 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -9,7 +9,9 @@ use rustc_hir::def_id::LocalDefId; use rustc_index::IndexVec; use rustc_index::bit_set::BitMatrix; use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; +use rustc_session::config::PackCoroutineLayout; use rustc_span::{Span, Symbol}; +use rustc_type_ir::data_structures::IndexMap; use super::{ConstValue, SourceInfo}; use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty}; @@ -55,6 +57,23 @@ pub struct CoroutineLayout<'tcx> { #[type_foldable(identity)] #[type_visitable(ignore)] pub storage_conflicts: BitMatrix, + + /// This map `A -> B` allows later MIR passes, error reporters + /// and layout calculator to relate saved locals `A` sourced from upvars + /// and locals `B` that upvars are moved into. + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub relocated_upvars: IndexMap, + + /// Coroutine layout packing + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub pack: PackCoroutineLayout, +} + +impl<'tcx> CoroutineLayout<'tcx> { + /// The initial state of a coroutine + pub const UNRESUMED: VariantIdx = VariantIdx::ZERO; } impl Debug for CoroutineLayout<'_> { @@ -80,6 +99,7 @@ impl Debug for CoroutineLayout<'_> { map.finish() }) .field("storage_conflicts", &self.storage_conflicts) + .field("relocated_upvars", &self.relocated_upvars) .finish() } } diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index f310e1e576250..f84f5765d8f87 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -157,13 +157,13 @@ impl<'tcx> PlaceTy<'tcx> { .get(f.index()) .copied() .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), - // Only prefix fields (upvars and current state) are - // accessible without a variant index. - ty::Coroutine(_, args) => { - args.as_coroutine().prefix_tys().get(f.index()).copied().unwrap_or_else(|| { - bug!("field {f:?} out of range of prefixes for {self_ty}") - }) - } + // Only upvars are accessible without a variant index. + ty::Coroutine(_, args) => args + .as_coroutine() + .upvar_tys() + .get(f.index()) + .copied() + .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), ty::Tuple(tys) => tys .get(f.index()) .copied() diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index c27d47fcc0d8d..d0187befd0b73 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -409,15 +409,14 @@ impl<'tcx> fmt::Display for Instance<'tcx> { // async_drop_in_place::coroutine's poll function (through FutureDropPollShim proxy) fn resolve_async_drop_poll<'tcx>(mut cor_ty: Ty<'tcx>) -> Instance<'tcx> { let first_cor = cor_ty; - let ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { + let &ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { bug!(); }; - let poll_def_id = *poll_def_id; let mut child_ty = cor_ty; loop { - if let ty::Coroutine(child_def, child_args) = child_ty.kind() { + if let &ty::Coroutine(child_def, child_args) = child_ty.kind() { cor_ty = child_ty; - if *child_def == poll_def_id { + if child_def == poll_def_id { child_ty = child_args.first().unwrap().expect_ty(); continue; } else { diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 507a25a432594..1b029c43bf0c6 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -952,9 +952,10 @@ where ), Variants::Multiple { tag, tag_field, .. } => { if FieldIdx::from_usize(i) == tag_field { - return TyMaybeWithLayout::TyAndLayout(tag_layout(tag)); + TyMaybeWithLayout::TyAndLayout(tag_layout(tag)) + } else { + TyMaybeWithLayout::Ty(args.as_coroutine().upvar_tys()[i]) } - TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i]) } }, diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index ce4de6b95e0bb..7015fe1fdc412 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -40,13 +40,14 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap}; use rustc_hir::definitions::DisambiguatorState; use rustc_hir::{LangItem, attrs as attr, find_attr}; use rustc_index::IndexVec; -use rustc_index::bit_set::BitMatrix; +use rustc_index::bit_set::{BitMatrix, DenseBitSet}; use rustc_macros::{ Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable, extension, }; use rustc_query_system::ich::StableHashingContext; use rustc_serialize::{Decodable, Encodable}; +use rustc_session::config::PackCoroutineLayout; pub use rustc_session::lint::RegisteredTools; use rustc_span::hygiene::MacroKind; use rustc_span::{DUMMY_SP, ExpnId, ExpnKind, Ident, Span, Symbol, sym}; @@ -63,6 +64,7 @@ pub use rustc_type_ir::solve::SizedTraitKind; pub use rustc_type_ir::*; #[allow(hidden_glob_reexports, unused_imports)] use rustc_type_ir::{InferCtxtLike, Interner}; +use smallvec::SmallVec; use tracing::{debug, instrument, trace}; pub use vtable::*; use {rustc_ast as ast, rustc_hir as hir}; @@ -113,7 +115,7 @@ pub use self::typeck_results::{ use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason}; use crate::metadata::ModChild; use crate::middle::privacy::EffectiveVisibilities; -use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, SourceInfo}; +use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, CoroutineSavedTy, SourceInfo}; use crate::query::{IntoQueryParam, Providers}; use crate::ty; use crate::ty::codec::{TyDecoder, TyEncoder}; @@ -1876,39 +1878,77 @@ impl<'tcx> TyCtxt<'tcx> { .ok_or_else(|| self.layout_error(LayoutError::Unknown(ty()))) } + /// Construct the trivial coroutine layout for async_drop of a coroutine + #[instrument(level = "debug", ret, skip(self))] + fn build_async_drop_trivial_coroutine_layout( + self, + def_id: DefId, + upvar_ty: Ty<'tcx>, + ) -> CoroutineLayout<'tcx> { + let upvar_tys = [upvar_ty]; + let nr_upvars = 1; + let span = self.def_span(def_id); + let source_info = SourceInfo::outermost(span); + let mut field_tys = IndexVec::new(); + let field_names = [None].into(); + let upvar_saved_locals: SmallVec<[_; 4]> = upvar_tys + .into_iter() + .map(|ty| field_tys.push(CoroutineSavedTy { ty, source_info, ignore_for_traits: true })) + .collect(); + // Even minimal, the trivial coroutine has 3 states (RESERVED_VARIANTS), + // so variant_fields and variant_source_info should have 3 elements. + let mut variant_fields: IndexVec> = + iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields[VariantIdx::ZERO].extend(upvar_saved_locals.clone()); + let variant_source_info: IndexVec = + iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + let mut storage_conflicts = BitMatrix::new(nr_upvars, nr_upvars); + if let Some(&first) = upvar_saved_locals.first() + && let Some(&last) = upvar_saved_locals.last() + { + let mut upvars = DenseBitSet::new_empty(nr_upvars); + upvars.insert_range(first..=last); + for &local in &upvar_saved_locals { + storage_conflicts.union_row_with(&upvars, local); + } + } + let relocated_upvars = Default::default(); + CoroutineLayout { + field_tys, + field_names, + variant_fields, + variant_source_info, + storage_conflicts, + relocated_upvars, + pack: PackCoroutineLayout::CapturesOnly, + } + } + /// Returns layout of a coroutine. Layout might be unavailable if the /// coroutine is tainted by errors. + #[instrument(level = "debug", skip(self), ret)] pub fn coroutine_layout( self, def_id: DefId, args: GenericArgsRef<'tcx>, ) -> Result<&'tcx CoroutineLayout<'tcx>, &'tcx LayoutError<'tcx>> { if self.is_async_drop_in_place_coroutine(def_id) { - // layout of `async_drop_in_place::{closure}` in case, - // when T is a coroutine, contains this internal coroutine's ptr in upvars - // and doesn't require any locals. Here is an `empty coroutine's layout` + // Layout of `async_drop_in_place::{closure}` when T is a coroutine + // It contains exactly one upvar which is a raw pointer to this inner coroutine + // and it does not suspend. let arg_cor_ty = args.first().unwrap().expect_ty(); - if arg_cor_ty.is_coroutine() { - let span = self.def_span(def_id); - let source_info = SourceInfo::outermost(span); - // Even minimal, empty coroutine has 3 states (RESERVED_VARIANTS), - // so variant_fields and variant_source_info should have 3 elements. - let variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let variant_source_info: IndexVec = - iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let proxy_layout = CoroutineLayout { - field_tys: [].into(), - field_names: [].into(), - variant_fields, - variant_source_info, - storage_conflicts: BitMatrix::new(0, 0), - }; - return Ok(self.arena.alloc(proxy_layout)); + if let &ty::Coroutine(_did, _args) = arg_cor_ty.kind() { + debug!("it is a trivial coroutine"); + let upvar_ty = Ty::new_mut_ptr(self, arg_cor_ty); + Ok(self + .arena + .alloc(self.build_async_drop_trivial_coroutine_layout(def_id, upvar_ty))) } else { + debug!("it is an async drop coroutine"); self.async_drop_coroutine_layout(def_id, args) } } else { + debug!("it is an ordinary coroutine"); self.ordinary_coroutine_layout(def_id, args) } } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index a3fdd4e35b6aa..670522a060cc2 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -150,13 +150,6 @@ impl<'tcx> ty::CoroutineArgs> { }) }) } - - /// This is the types of the fields of a coroutine which are not stored in a - /// variant. - #[inline] - fn prefix_tys(self) -> &'tcx List> { - self.upvar_tys() - } } #[derive(Debug, Copy, Clone, HashStable, TypeFoldable, TypeVisitable)] diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index c136df812a3f0..16559a735654c 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -52,7 +52,9 @@ mod by_move_body; mod drop; -use std::{iter, ops}; +mod relocate_upvars; + +use std::ops::Deref; pub(super) use by_move_body::coroutine_by_move_body_def_id; use drop::{ @@ -60,14 +62,16 @@ use drop::{ create_coroutine_drop_shim_proxy_async, elaborate_coroutine_drops, expand_async_drops, has_expandable_async_drops, insert_clean_drop, }; +pub(super) use relocate_upvars::RelocateUpvars; use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::unord::UnordMap; use rustc_errors::pluralize; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_hir::{CoroutineDesugaring, CoroutineKind}; use rustc_index::bit_set::{BitMatrix, DenseBitSet, GrowableBitSet}; -use rustc_index::{Idx, IndexVec}; +use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::util::Discr; @@ -82,13 +86,15 @@ use rustc_mir_dataflow::impls::{ use rustc_mir_dataflow::{ Analysis, Results, ResultsCursor, ResultsVisitor, visit_reachable_results, }; +use rustc_session::config::PackCoroutineLayout; use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::source_map::dummy_spanned; use rustc_span::symbol::sym; -use rustc_span::{DUMMY_SP, Span}; +use rustc_span::{DUMMY_SP, Span, Symbol, kw}; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::infer::TyCtxtInferExt as _; use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode, ObligationCtxt}; +use smallvec::{SmallVec, smallvec}; use tracing::{debug, instrument, trace}; use crate::deref_separator::deref_finder; @@ -102,6 +108,8 @@ struct RenameLocalVisitor<'tcx> { tcx: TyCtxt<'tcx>, } +const VARIANT_UNRESUMED: VariantIdx = VariantIdx::from_usize(CoroutineArgs::UNRESUMED); + impl<'tcx> MutVisitor<'tcx> for RenameLocalVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx @@ -198,8 +206,11 @@ struct TransformVisitor<'tcx> { // A map from a suspension point in a block to the locals which have live storage at that point storage_liveness: IndexVec>>, + // A rev-lookup from basic blocks with yielding terminator to the suspension point index, + suspension_point_at_block: UnordMap, + // A list of suspension points, generated during the transform - suspension_points: Vec>, + suspension_points: IndexVec>>, // The set of locals that have no `StorageLive`/`StorageDead` annotations. always_live_locals: DenseBitSet, @@ -402,6 +413,16 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // Replace an Local in the remap with a coroutine struct access if let Some(&Some((ty, variant_index, idx))) = self.remap.get(place.local) { replace_base(place, self.make_field(variant_index, idx, ty), self.tcx); + } else if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = *place + && let [first @ ProjectionElem::Field(..), rest @ ..] = &**projection + { + let projections: Vec<_> = [ProjectionElem::Downcast(None, VARIANT_UNRESUMED), *first] + .into_iter() + .chain(rest.iter().copied()) + .collect(); + let new_place = + Place::from(ty::CAPTURE_STRUCT_LOCAL).project_deeper(&projections, self.tcx); + *place = new_place; } } @@ -430,8 +451,9 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // We must assign the value first in case it gets declared dead below self.make_state(v, source_info, is_return, &mut data.statements); let state = if let Some((resume, mut resume_arg)) = resume { - // Yield - let state = CoroutineArgs::RESERVED_VARIANTS + self.suspension_points.len(); + // This is a `yield` + let suspension_point_idx = *self.suspension_point_at_block.get(&block).unwrap(); + let state = CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize(); // The resume arg target location might itself be remapped if its base local is // live across a yield. @@ -453,12 +475,8 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { } } - self.suspension_points.push(SuspensionPoint { - state, - resume, - resume_arg, - drop, - storage_liveness, + self.suspension_points.get_or_insert_with(suspension_point_idx, || { + SuspensionPoint { state, resume, resume_arg, drop, storage_liveness } }); VariantIdx::new(state) @@ -484,19 +502,19 @@ fn make_aggregate_adt<'tcx>( } fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let ref_coroutine_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, coroutine_ty); // Replace the by value coroutine argument - body.local_decls.raw[1].ty = ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = ref_coroutine_ty; // Add a deref to accesses of the coroutine state SelfArgVisitor::new(tcx, ProjectionElem::Deref).visit_body(body); } fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let ref_coroutine_ty = body.local_decls.raw[1].ty; + let ref_coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let pin_did = tcx.require_lang_item(LangItem::Pin, body.span); let pin_adt_ref = tcx.adt_def(pin_did); @@ -504,7 +522,7 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body let pin_ref_coroutine_ty = Ty::new_adt(tcx, pin_adt_ref, args); // Replace the by ref coroutine argument - body.local_decls.raw[1].ty = pin_ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = pin_ref_coroutine_ty; // Add the Pin field access to accesses of the coroutine state SelfArgVisitor::new(tcx, ProjectionElem::Field(FieldIdx::ZERO, ref_coroutine_ty)) @@ -642,15 +660,20 @@ fn transform_gen_context<'tcx>(body: &mut Body<'tcx>) { body.arg_count = 1; } +#[derive(Debug)] struct LivenessInfo { /// Which locals are live across any suspension point. saved_locals: CoroutineSavedLocals, + /// Always live locals + always_live_locals: DenseBitSet, + /// The set of saved locals live at each suspension point. - live_locals_at_suspension_points: Vec>, + live_locals_at_suspension_points: + IndexVec>, /// Parallel vec to the above with SourceInfo for each yield terminator. - source_info_at_suspension_points: Vec, + source_info_at_suspension_points: IndexVec, /// For every saved local, the set of other saved locals that are /// storage-live at the same time as this local. We cannot overlap locals in @@ -660,6 +683,14 @@ struct LivenessInfo { /// For every suspending block, the locals which are storage-live across /// that suspension point. storage_liveness: IndexVec>>, + + /// A rev-lookup of basic blocks to the suspension point index + suspension_point_at_block: UnordMap, +} + +rustc_index::newtype_index! { + #[debug_format = "suspend_{}"] + struct SuspensionPointIdx {} } /// Computes which locals have to be stored in the state-machine for the @@ -673,12 +704,12 @@ struct LivenessInfo { fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - always_live_locals: &DenseBitSet, + always_live_locals: DenseBitSet, movable: bool, ) -> LivenessInfo { // Calculate when MIR locals have live storage. This gives us an upper bound of their // lifetimes. - let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(always_live_locals)) + let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(&always_live_locals)) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); @@ -711,11 +742,13 @@ fn locals_live_across_suspend_points<'tcx>( MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("coroutine")).into_results_cursor(body); let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks); - let mut live_locals_at_suspension_points = Vec::new(); - let mut source_info_at_suspension_points = Vec::new(); + let mut live_locals_at_suspension_points = IndexVec::::default(); + let mut source_info_at_suspension_points = IndexVec::default(); let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len()); + let mut suspension_point_at_block = UnordMap::default(); - for (block, data) in body.basic_blocks.iter_enumerated() { + for &block in body.basic_blocks.reverse_postorder() { + let data = &body.basic_blocks[block]; if let TerminatorKind::Yield { .. } = data.terminator().kind { let loc = Location { block, statement_index: data.statements.len() }; @@ -749,7 +782,7 @@ fn locals_live_across_suspend_points<'tcx>( live_locals.intersect(requires_storage_cursor.get()); // The coroutine argument is ignored. - live_locals.remove(SELF_ARG); + live_locals.remove(ty::CAPTURE_STRUCT_LOCAL); debug!("loc = {:?}, live_locals = {:?}", loc, live_locals); @@ -757,8 +790,9 @@ fn locals_live_across_suspend_points<'tcx>( // any suspension points live_locals_at_any_suspension_point.union(&live_locals); - live_locals_at_suspension_points.push(live_locals); + let idx = live_locals_at_suspension_points.push(live_locals); source_info_at_suspension_points.push(data.terminator().source_info); + suspension_point_at_block.insert(block, idx); } } @@ -782,10 +816,12 @@ fn locals_live_across_suspend_points<'tcx>( LivenessInfo { saved_locals, + always_live_locals, live_locals_at_suspension_points, source_info_at_suspension_points, storage_conflicts, storage_liveness: storage_liveness_map, + suspension_point_at_block, } } @@ -794,6 +830,7 @@ fn locals_live_across_suspend_points<'tcx>( /// `CoroutineSavedLocal` is indexed in terms of the elements in this set; /// i.e. `CoroutineSavedLocal::new(1)` corresponds to the second local /// included in this set. +#[derive(Debug)] struct CoroutineSavedLocals(DenseBitSet); impl CoroutineSavedLocals { @@ -826,7 +863,7 @@ impl CoroutineSavedLocals { } } -impl ops::Deref for CoroutineSavedLocals { +impl Deref for CoroutineSavedLocals { type Target = DenseBitSet; fn deref(&self) -> &Self::Target { @@ -945,25 +982,46 @@ impl StorageConflictVisitor<'_, '_> { } } +#[derive(Debug, Copy, Clone)] +struct UpvarInfo { + name: Symbol, + span: Span, +} + +#[instrument[level = "debug", skip(body), fields(body = ?body.source)]] fn compute_layout<'tcx>( liveness: LivenessInfo, body: &Body<'tcx>, + upvar_tys: &[Ty<'tcx>], + upvar_infos: &[UpvarInfo], + pack: PackCoroutineLayout, + local_upvar_map: &IndexSlice>, ) -> ( IndexVec, VariantIdx, FieldIdx)>>, CoroutineLayout<'tcx>, IndexVec>>, + UnordMap, ) { let LivenessInfo { saved_locals, + always_live_locals, live_locals_at_suspension_points, source_info_at_suspension_points, storage_conflicts, storage_liveness, + suspension_point_at_block, } = liveness; + // We need to later establish the map between upvars in UNRESUMED and locals in other states. + let local_upvar_map: UnordMap<_, _> = local_upvar_map + .iter_enumerated() + .filter_map(|(field, local)| local.map(|local| (local, field))) + .collect(); + // Gather live local types and their indices. let mut locals = IndexVec::::new(); let mut tys = IndexVec::::new(); + let mut saved_local_upvar_map = UnordMap::default(); for (saved_local, local) in saved_locals.iter_enumerated() { debug!("coroutine saved local {:?} => {:?}", saved_local, local); @@ -991,7 +1049,52 @@ fn compute_layout<'tcx>( debug!(?decl); tys.push(decl); + + if let Some(&field) = local_upvar_map.get(&local) { + saved_local_upvar_map.insert(field, saved_local); + } } + // These are the "saved locals" sourced from the UNRESUMED state. + let upvar_saved_locals: IndexVec = upvar_tys + .iter() + .zip(upvar_infos) + .map(|(&ty, info)| { + tys.push(CoroutineSavedTy { + ty, + source_info: SourceInfo::outermost(info.span), + ignore_for_traits: false, + }) + }) + .collect(); + debug!(?upvar_saved_locals); + let storage_conflicts = if let Some(&first) = upvar_saved_locals.raw.first() + && let Some(&last) = upvar_saved_locals.raw.last() + { + let mut enlarged_storage_conflicts = BitMatrix::new(tys.len(), tys.len()); + let mut upvars = DenseBitSet::new_empty(tys.len()); + let mut ineligibles = upvars.clone(); + upvars.insert_range(first..=last); + for (saved_local, local) in saved_locals.iter_enumerated() { + if always_live_locals.contains(local) { + ineligibles.insert(saved_local); + } + } + upvars.union(&ineligibles); + for row in storage_conflicts.rows() { + for column in storage_conflicts.iter(row) { + enlarged_storage_conflicts.insert(row, column); + } + } + for &upvar in &upvar_saved_locals { + enlarged_storage_conflicts.union_row_with(&upvars, upvar); + } + for ineligible in ineligibles.iter() { + enlarged_storage_conflicts.union_row_with(&upvars, ineligible); + } + enlarged_storage_conflicts + } else { + storage_conflicts + }; // Leave empty variants for the UNRESUMED, RETURNED, and POISONED states. // In debuginfo, these will correspond to the beginning (UNRESUMED) or end @@ -1008,12 +1111,14 @@ fn compute_layout<'tcx>( // Build the coroutine variant field list. // Create a map from local indices to coroutine struct indices. + let variant_fields: [_; CoroutineArgs::RESERVED_VARIANTS] = + [upvar_saved_locals.clone(), IndexVec::new(), IndexVec::new()]; let mut variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields.into_iter().collect(); let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); - for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { + for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter_enumerated() { let variant_index = - VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); + VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize()); let mut fields = IndexVec::new(); for (idx, saved_local) in live_locals.iter().enumerate() { fields.push(saved_local); @@ -1029,29 +1134,43 @@ fn compute_layout<'tcx>( } debug!("coroutine variant_fields = {:?}", variant_fields); debug!("coroutine storage_conflicts = {:#?}", storage_conflicts); + debug!(remap = ?remap.debug_map_view_compact()); + debug!(locals = ?locals.debug_map_view()); let mut field_names = IndexVec::from_elem(None, &tys); for var in &body.var_debug_info { - let VarDebugInfoContents::Place(place) = &var.value else { continue }; - let Some(local) = place.as_local() else { continue }; - let Some(&Some((_, variant, field))) = remap.get(local) else { - continue; - }; - - let saved_local = variant_fields[variant][field]; - field_names.get_or_insert_with(saved_local, || var.name); + debug!(?var); + if let VarDebugInfoContents::Place(place) = &var.value + && let Some(local) = place.local_or_deref_local() + && let Some(&Some((_, variant, field))) = remap.get(local) + { + let saved_local = variant_fields[variant][field]; + field_names.get_or_insert_with(saved_local, || var.name); + } + } + for (capture, &saved_local) in upvar_infos.iter().zip(&upvar_saved_locals) { + field_names.get_or_insert_with(saved_local, || capture.name); } + debug!(field_names = ?field_names.debug_map_view()); + let relocated_upvars = upvar_saved_locals + .iter_enumerated() + .filter_map(|(field, &source)| { + saved_local_upvar_map.get(&field).map(|&dest| (source, dest)) + }) + .collect(); let layout = CoroutineLayout { field_tys: tys, field_names, variant_fields, variant_source_info, storage_conflicts, + relocated_upvars, + pack, }; debug!(?layout); - (remap, layout, storage_liveness) + (remap, layout, storage_liveness, suspension_point_at_block) } /// Replaces the entry point of `body` with a block that switches on the coroutine discriminant and @@ -1325,6 +1444,7 @@ fn create_cases<'tcx>( .suspension_points .iter() .filter_map(|point| { + let Some(point) = point else { bug!("all suspension points must be resolved now") }; // Find the target for this suspension point, if applicable operation.target_block(point).map(|target| { let mut statements = Vec::new(); @@ -1369,14 +1489,20 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( def_id: LocalDefId, ) -> Option> { let (body, _) = tcx.mir_promoted(def_id); - let body = body.borrow(); - let body = &*body; + let body = &*body.borrow(); // The first argument is the coroutine type passed by value let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; - let movable = match *coroutine_ty.kind() { - ty::Coroutine(def_id, _) => tcx.coroutine_movability(def_id) == hir::Movability::Movable, + let (movable, upvar_tys, upvar_infos) = match *coroutine_ty.kind() { + ty::Coroutine(def_id, args) => ( + matches!(tcx.coroutine_movability(def_id), hir::Movability::Movable), + args.as_coroutine().upvar_tys(), + tcx.closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ), ty::Error(_) => return None, _ => span_bug!(body.span, "unexpected coroutine type {}", coroutine_ty), }; @@ -1384,13 +1510,22 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( // The witness simply contains all locals live across suspend points. let always_live_locals = always_storage_live_locals(body); - let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + debug!(?always_live_locals); + let liveness_info = locals_live_across_suspend_points(tcx, body, always_live_locals, movable); // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (_, coroutine_layout, _) = compute_layout(liveness_info, body); - + let (_, coroutine_layout, _, _) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + // Layout packing only happens at state transformation + // and we do not reflect this in analysis + PackCoroutineLayout::No, + IndexSlice::empty(), + ); check_suspend_tys(tcx, &coroutine_layout, body); check_field_tys_sized(tcx, &coroutine_layout, def_id); @@ -1448,20 +1583,51 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { let old_ret_ty = body.return_ty(); assert!(body.coroutine_drop().is_none() && body.coroutine_drop_async().is_none()); + let mut local_upvar_map = Default::default(); + RelocateUpvars::new(!matches!( + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + rustc_session::config::PackCoroutineLayout::No + )) + .run(tcx, body, &mut local_upvar_map); if let Some(dumper) = MirDumper::new(tcx, "coroutine_before", body) { dumper.dump_mir(body); } // The first argument is the coroutine type passed by value - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let coroutine_kind = body.coroutine_kind().unwrap(); // Get the discriminant type and args which typeck computed - let ty::Coroutine(_, args) = coroutine_ty.kind() else { + let ty::Coroutine(def_id, args) = coroutine_ty.kind() else { tcx.dcx().span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}")); }; - let discr_ty = args.as_coroutine().discr_ty(tcx); + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![UpvarInfo { name: kw::SelfLower, span: DUMMY_SP }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; + let coroutine_args = args.as_coroutine(); + let discr_ty = coroutine_args.discr_ty(tcx); + let upvar_tys = coroutine_args.upvar_tys(); let new_ret_ty = match coroutine_kind { CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => { @@ -1520,9 +1686,9 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { } let always_live_locals = always_storage_live_locals(body); - let movable = coroutine_kind.movability() == hir::Movability::Movable; + let movable = matches!(coroutine_kind.movability(), hir::Movability::Movable); let liveness_info = - locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + locals_live_across_suspend_points(tcx, body, always_live_locals.clone(), movable); if tcx.sess.opts.unstable_opts.validate_mir { let mut vis = EnsureCoroutineFieldAssignmentsNeverAlias { @@ -1537,7 +1703,14 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); + let (remap, layout, storage_liveness, suspension_point_at_block) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + &local_upvar_map, + ); let can_return = can_return(tcx, body, body.typing_env(tcx)); @@ -1552,11 +1725,12 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { remap, storage_liveness, always_live_locals, - suspension_points: Vec::new(), + suspension_points: IndexVec::default(), old_ret_local, discr_ty, old_ret_ty, old_yield_ty, + suspension_point_at_block, }; transform.visit_body(body); diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 951ff69c19e3e..06e0cfc08e57c 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -69,7 +69,6 @@ use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::steal::Steal; -use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; @@ -78,8 +77,18 @@ use rustc_middle::bug; use rustc_middle::hir::place::{Projection, ProjectionKind}; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, MirDumper}; +use rustc_middle::ty::data_structures::IndexMap; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; +struct CaptureInfo<'tcx> { + /// Field index of the capture in the parent coroutine structure + remapped_idx: FieldIdx, + /// Type of the capture in the parent coroutine structure + remapped_ty: Ty<'tcx>, + peel_deref: bool, + bridging_projections: Vec>, +} + pub(crate) fn coroutine_by_move_body_def_id<'tcx>( tcx: TyCtxt<'tcx>, coroutine_def_id: LocalDefId, @@ -126,23 +135,27 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( .tuple_fields() .len(); - let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures( + let field_remapping: IndexMap<_, _> = ty::analyze_coroutine_closure_captures( tcx.closure_captures(parent_def_id).iter().copied(), tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(), |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { // Store this set of additional projections (fields and derefs). // We need to re-apply them later. - let mut child_precise_captures = - child_capture.place.projections[parent_capture.place.projections.len()..].to_vec(); + let child_precise_captures = child_capture.place.projections + [parent_capture.place.projections.len()..] + .iter() + .copied(); // If the parent capture is by-ref, then we need to apply an additional // deref before applying any further projections to this place. - if parent_capture.is_by_ref() { - child_precise_captures.insert( - 0, - Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }, - ); - } + let bridging_projections = if parent_capture.is_by_ref() { + [Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }] + .into_iter() + .chain(child_precise_captures) + .collect() + } else { + child_precise_captures.collect() + }; // If the child capture is by-ref, then we need to apply a "ref" // projection (i.e. `&`) at the end. But wait! We don't have that // as a projection kind. So instead, we can apply its dual and @@ -168,8 +181,8 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( // Finally, store the type of the parent's captured place. We need // this when building the field projection in the MIR body later on. - let mut parent_capture_ty = parent_capture.place.ty(); - parent_capture_ty = match parent_capture.info.capture_kind { + let parent_capture_ty = parent_capture.place.ty(); + let remapped_ty = match parent_capture.info.capture_kind { ty::UpvarCapture::ByValue | ty::UpvarCapture::ByUse => parent_capture_ty, ty::UpvarCapture::ByRef(kind) => Ty::new_ref( tcx, @@ -181,19 +194,19 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( Some(( FieldIdx::from_usize(child_field_idx + num_args), - ( - FieldIdx::from_usize(parent_field_idx + num_args), - parent_capture_ty, + CaptureInfo { + remapped_idx: FieldIdx::from_usize(parent_field_idx + num_args), + remapped_ty, peel_deref, - child_precise_captures, - ), + bridging_projections, + }, )) }, ) .flatten() .collect(); - if coroutine_kind == ty::ClosureKind::FnOnce { + if matches!(coroutine_kind, ty::ClosureKind::FnOnce) { assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); // The by-move body is just the body :) return coroutine_def_id.to_def_id(); @@ -212,6 +225,9 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( ); let mut by_move_body = body.clone(); + if let Some(mir_dumper) = MirDumper::new(tcx, "built", &body) { + mir_dumper.dump_mir(&body); + } MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); // This path is unique since we're in a query so we'll only be called once with `parent_def_id` @@ -254,7 +270,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - field_remapping: UnordMap, bool, Vec>)>, + field_remapping: IndexMap>, by_move_coroutine_ty: Ty<'tcx>, } @@ -275,8 +291,12 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { if place.local == ty::CAPTURE_STRUCT_LOCAL && let Some((&mir::ProjectionElem::Field(idx, _), projection)) = place.projection.split_first() - && let Some(&(remapped_idx, remapped_ty, peel_deref, ref bridging_projections)) = - self.field_remapping.get(&idx) + && let Some(&CaptureInfo { + remapped_idx, + remapped_ty, + peel_deref, + ref bridging_projections, + }) = self.field_remapping.get(&idx) { // As noted before, if the parent closure captures a field by value, and // the child captures a field by ref, then for the by-move body we're @@ -353,7 +373,7 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { local: ty::CAPTURE_STRUCT_LOCAL, projection: [mir::ProjectionElem::Field(idx, _)], } = place.as_ref() - && let Some(&(_, _, true, _)) = self.field_remapping.get(&idx) + && let Some(CaptureInfo { peel_deref: true, .. }) = self.field_remapping.get(idx) { statement.kind = mir::StatementKind::Nop; } diff --git a/compiler/rustc_mir_transform/src/coroutine/drop.rs b/compiler/rustc_mir_transform/src/coroutine/drop.rs index fd2d8b2b0563e..12f5da3903a3a 100644 --- a/compiler/rustc_mir_transform/src/coroutine/drop.rs +++ b/compiler/rustc_mir_transform/src/coroutine/drop.rs @@ -231,8 +231,14 @@ pub(super) fn has_expandable_async_drops<'tcx>( if body[bb].is_cleanup { continue; } - let TerminatorKind::Drop { place, target: _, unwind: _, replace: _, drop: _, async_fut } = - body[bb].terminator().kind + let TerminatorKind::Drop { + place, + target: _, + unwind: _, + replace: _, + drop: _, + async_fut: Some(_), + } = body[bb].terminator().kind else { continue; }; @@ -240,12 +246,9 @@ pub(super) fn has_expandable_async_drops<'tcx>( if place_ty == coroutine_ty { continue; } - if async_fut.is_none() { - continue; - } return true; } - return false; + false } /// Expand Drop terminator for async drops into mainline poll-switch and dropline poll-switch @@ -508,7 +511,8 @@ pub(super) fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body elaborate_drop( &mut elaborator, *source_info, - Place::from(SELF_ARG), + Place::from(SELF_ARG) + .project_deeper(&[ProjectionElem::Downcast(None, VARIANT_UNRESUMED)], tcx), (), *target, unwind, diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs new file mode 100644 index 0000000000000..96fece430410d --- /dev/null +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -0,0 +1,557 @@ +//! MIR rewrite pass to relocate upvars into native locals in the coroutine body +//! +//! # Summary +//! The current contract of coroutine upvars is as follows. +//! Coroutines are constructed, initially in state UNRESUMED, by copying or moving +//! captures into the `struct`-fields, which are also called prefix fields, +//! taking necessary references as per capture specification. +//! +//! ```text +//! Low address High address +//! ┌─────────┬─────────┬─────┬─────────────────────┬───────────────────────────────────────────────────────┬──────────────┐ +//! │ │ │ │ │ │ │ +//! │ Upvar 1 │ Upvar 2 │ ... │ Coroutine State Tag │ Ineligibles, aka. saved locals alive across 2+ states │ Other states │ +//! │ │ │ │ │ │ │ +//! └─────────┴─────────┴─────┴─────────────────────┴───────────────────────────────────────────────────────┴──────────────┘ +//! ``` +//! +//! In case some upvars are large and short-lived, the classic layout scheme can be wasteful. +//! One way to reduce the memory footprint is to +//! +//! This pass performs the following transformations. +//! 1. It generates a fresh batch of locals for each captured upvars. +//! +//! For each upvar, whether used or not, a fresh local is created with the same type. +//! The types respect the nature of the captures, being by-ref, by-ref-mut or by-value. +//! This is reflected in the results in the upvar analysis conducted in the HIR type-checking phase. +//! +//! 2. It replaces the places pointing into those upvars with places pointing into those locals instead +//! +//! Each place that starts with access into the coroutine structure `_1` is replaced with the fresh local as +//! the base. +//! Suppose we are to lower this coroutine into the MIR. +//! ```ignore (illustrative) +//! let mut captured = None; +//! let _ = #[coroutine] || { +//! yield (); +//! if let Some(inner) = &mut captured { +//! *inner = 42i32; // (*) +//! } +//! }; +//! ``` +//! `captured` is the only capture, whose mutable borrow is formally allotted to the first field `_1.0: &mut i32`. +//! The highlighted line `(*)` should be lowered, roughly, into MIR `(*_1.0) = const 42i32;`. +//! Now, by application of this pass, we create a new local `_4: &mut i32` and we perform the following +//! code transformation. +//! +//! A new block is constructed to just perform the relocation of this mutable borrow. +//! This block is inserted to the very beginning of the coroutine body control flow, +//! so that this is executed before any proper coroutine code as it transits from `UNRESUME` state to +//! any other state. +//! This "prologue" will look like the following. +//! ```ignore (illustrative) +//! StorageLive(_5); +//! StorageLive(_4); +//! _5 = move (_1.0); +//! _4 = move (_5); +//! StorageDead(_5); +//! ``` +//! Note that we also create a trampoline local `_5` of the same type. +//! +//! ### Intricacy around the trampoline local +//! The reason that we need the trampolines is because state transformation and coroutine +//! layout calculation is not aware of potential storage conflict between captures as struct fields +//! and other saved locals. +//! The only guarantee that we can work with is one where any coroutine layout calculator respects +//! the storage conflict contracts between *MIR locals*. +//! It is known that calculators do not consider struct fields, where captures reside, as MIR locals. +//! This is the source of potential memory overlaps. +//! For instance, in a hypothetical situation, +//! - `_1.0` is relocated to `_4`; +//! - `_1.1` is relocated to `_6`; +//! - `_4` and `_6` remains live at one of the first suspension state; +//! - `_4` occupies the same offset of `_1.1` and `_6` occupies the same offset of `_1.0` +//! as decided by some layout calculator; +//! In this scenario, without trampolining, the relocations introduce undefined behaviour. +//! +//! As a proposal for a future design, it is best that coroutine captures receive their own +//! MIR locals, possibly in a form of "function arguments" like `_1` itself. +//! The trampolining transformation already attempts to restore the semantics of MIR locals to +//! these captures and promoting them to "arguments" would make MIR safer to handle. +//! +//! One should note that this phase assumes that the initial built MIR respects the nature of captures. +//! For instance, if the upvar `_1.4` is instead a by-ref-mut capture of a value of type `T`, +//! this phase assumes that all access correctly built as operating on the place `(*_1.4)`. +//! Based on the assumption, this phase replaces `_1.4` with a fresh local `_34: &mut T` and +//! the correctness is still upheld. +//! +//! 3. It assembles an prologue to replace the current entry block. +//! +//! This prologue block transfers every captured upvar into its corresponding fresh local, *via scratch locals*. +//! The upvars are first completely moved into the scratch locals in batch, and then moved into the destination +//! locals in batch. +//! The reason is that it is possible that coroutine layout may change and the source memory location of +//! an upvar may not necessarily be mapped exactly to the same place as in the `UNRESUMED` state. +//! This is very possible, because the coroutine layout scheme at this moment remains opaque, +//! other than the contract that a saved local has a stable internal offset throughout its liveness span. +//! +//! While the current coroutine layout ensures that the same saved local has stable offsets throughout its lifetime, +//! technically the upvar in `UNRESUMED` state and their fresh locals are different saved locals. +//! This scratch locals re-establish safety so that the correct data permutation can take place, +//! when a future coroutine layout calculator sees the permutation fit. + +use std::borrow::Cow; + +use rustc_abi::FieldIdx; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::{IndexSlice, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::{ + self, BasicBlock, BasicBlockData, Body, Local, MirDumper, Place, ProjectionElem, START_BLOCK, + SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, UnwindAction, +}; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_mir_dataflow::Analysis; +use rustc_mir_dataflow::impls::{MaybeStorageLive, always_storage_live_locals}; +use rustc_span::{DUMMY_SP, Ident, Span, Symbol}; +use smallvec::{SmallVec, smallvec}; +use tracing::{debug, instrument}; + +use crate::pass_manager::MirPass; +use crate::patch::MirPatch; + +pub(crate) struct RelocateUpvars(bool); + +impl RelocateUpvars { + pub(crate) fn new(do_relocate: bool) -> Self { + Self(do_relocate) + } +} + +struct UpvarSubstitution<'tcx> { + /// Newly minted local into which the upvar is moved + local: Local, + /// The temporary local that the prologue will permute the upvars with + reloc: Local, + /// Place into the capture structure where this upvar is found + upvar_place: Place<'tcx>, + /// The span of the captured upvar from the parent body + span: Span, + /// Name of the upvar + name: Symbol, +} + +struct SubstituteUpvarVisitor<'tcx, 'a> { + tcx: TyCtxt<'tcx>, + mappings: &'a IndexSlice>, +} + +impl<'tcx, 'a> MutVisitor<'tcx> for SubstituteUpvarVisitor<'tcx, 'a> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_place( + &mut self, + place: &mut Place<'tcx>, + _context: mir::visit::PlaceContext, + location: mir::Location, + ) { + if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = place + && let [ProjectionElem::Field(field_idx, _ty), rest @ ..] = &***projection + { + let Some(&UpvarSubstitution { local, .. }) = self.mappings.get(*field_idx) else { + bug!( + "SubstituteUpvar: found {field_idx:?} @ {location:?} but there is no upvar for it" + ) + }; + let new_place = Place::from(local); + let new_place = new_place.project_deeper(rest, self.tcx); + *place = new_place; + } + } + + fn visit_terminator( + &mut self, + terminator: &mut mir::Terminator<'tcx>, + location: mir::Location, + ) { + if let TerminatorKind::Drop { place, .. } = &terminator.kind + && let Some(ty::CAPTURE_STRUCT_LOCAL) = place.as_local() + { + // This is a drop on the whole coroutine state, which we will processed later + return; + } + self.super_terminator(terminator, location) + } +} + +#[derive(Debug, Clone, Copy)] +struct RelocationInfo { + ident: Option, + immutable: bool, + by_ref: bool, +} + +impl RelocateUpvars { + #[instrument(level = "debug", skip_all, fields(def_id = ?body.source))] + pub(crate) fn run<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + local_upvar_map: &mut IndexVec>, + ) { + if !self.0 { + debug!("relocate upvar is set to no-op"); + return; + } + if body.yield_ty().is_none() { + // It fails the litmus test as a coroutine + debug!("not passing litmus test"); + return; + } + + // The first argument is the coroutine type passed by value + let coroutine_ty = if let Some(decl) = body.local_decls.get(ty::CAPTURE_STRUCT_LOCAL) { + decl.ty + } else { + debug!("not coroutine ty, skipping"); + return; + }; + + // We only care when there is at least one upvar + let (def_id, upvar_tys) = if let ty::Coroutine(def_id, args) = *coroutine_ty.kind() { + let args = args.as_coroutine(); + (def_id, args.upvar_tys()) + } else { + debug!("not coroutine ty again, skipping"); + return; + }; + if upvar_tys.is_empty() { + debug!("no upvar, skipping"); + return; + } + + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![RelocationInfo { ident: None, immutable: true, by_ref: false }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| RelocationInfo { + ident: Some(Ident::new(info.to_symbol(), info.var_ident.span)), + immutable: matches!(info.mutability, ty::Mutability::Not), + by_ref: matches!(info.info.capture_kind, ty::UpvarCapture::ByRef(..)), + }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; + + if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { + mir_dumper.set_disambiguator(&"before").dump_mir(body); + } + + let mut substitution_mapping = IndexVec::new(); + let mut patch = MirPatch::new(body); + for (field_idx, (upvar_ty, &captured)) in upvar_tys.iter().zip(&upvar_infos).enumerate() { + let span = captured.ident.map_or(DUMMY_SP, |ident| ident.span); + let name = if let Some(ident) = captured.ident { + ident.name + } else { + Symbol::intern(&format!("_{}", field_idx)) + }; + + let immutable = + if captured.immutable { mir::Mutability::Not } else { mir::Mutability::Mut }; + let local = patch.new_local_with_info( + upvar_ty, + span, + mir::LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm { + binding_mode: rustc_ast::BindingMode( + if captured.by_ref { + rustc_ast::ByRef::Yes(immutable) + } else { + rustc_ast::ByRef::No + }, + immutable, + ), + opt_ty_info: None, + opt_match_place: None, + pat_span: span, + })), + captured.immutable, + ); + let reloc = patch.new_local_with_info(upvar_ty, span, mir::LocalInfo::Boring, true); + + let field_idx = FieldIdx::from_usize(field_idx); + let upvar_place = Place::from(ty::CAPTURE_STRUCT_LOCAL) + .project_deeper(&[ProjectionElem::Field(field_idx, upvar_ty)], tcx); + + substitution_mapping.push(UpvarSubstitution { local, reloc, upvar_place, span, name }); + } + patch.apply(body); + local_upvar_map.extend(substitution_mapping.iter().map(|sub| Some(sub.local))); + SubstituteUpvarVisitor { tcx, mappings: &substitution_mapping }.visit_body(body); + + rewrite_drop_coroutine_struct(tcx, body, &substitution_mapping); + insert_substitution_prologue(body, &substitution_mapping); + patch_missing_storage_deads(tcx, body, &substitution_mapping); + hydrate_var_debug_info(body, &substitution_mapping); + if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { + mir_dumper.set_disambiguator(&"after").dump_mir(body); + } + } +} + +impl<'tcx> MirPass<'tcx> for RelocateUpvars { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + self.run(tcx, body, &mut Default::default()) + } + + fn is_required(&self) -> bool { + self.0 + } +} + +fn rewrite_one_drop_coroutine_struct<'tcx>( + tcx: TyCtxt<'tcx>, + patch: &mut MirPatch<'tcx>, + body: &Body<'tcx>, + block: BasicBlock, + substitution_mapping: &IndexSlice>, +) { + let data = &body.basic_blocks[block]; + let source_info = data.terminator().source_info; + let TerminatorKind::Drop { + place: _, + mut target, + mut unwind, + replace, + drop: dropline, + async_fut, + } = data.terminator().kind + else { + unreachable!("unexpected terminator {:?}", data.terminator().kind) + }; + let mut cleanup = match unwind { + UnwindAction::Cleanup(tgt) => tgt, + UnwindAction::Continue => patch.resume_block(), + UnwindAction::Unreachable => patch.unreachable_cleanup_block(), + UnwindAction::Terminate(reason) => patch.terminate_block(reason), + }; + let typing_env = ty::TypingEnv::post_analysis(tcx, body.source.def_id()); + for &UpvarSubstitution { local, .. } in substitution_mapping { + let place = local.into(); + let mut unwind_one = patch.new_block(BasicBlockData::new_stmts( + vec![Statement::new(source_info, StatementKind::StorageDead(local))], + Some(Terminator { + source_info, + kind: TerminatorKind::Goto { + target: if data.is_cleanup { target } else { cleanup }, + }, + }), + true, + )); + let needs_drop = body.local_decls[local].ty.needs_drop(tcx, typing_env); + let kind = if needs_drop { + TerminatorKind::Drop { + place, + target: unwind_one, + unwind: UnwindAction::Terminate(mir::UnwindTerminateReason::InCleanup), + replace, + drop: None, + async_fut, + } + } else { + TerminatorKind::Goto { target: unwind_one } + }; + unwind_one = patch.new_block(BasicBlockData::new_stmts( + vec![], + Some(Terminator { source_info, kind }), + true, + )); + if data.is_cleanup { + unwind = UnwindAction::Cleanup(unwind_one); + cleanup = unwind_one; + target = unwind_one; + } else { + let mut drop_one = patch.new_block(BasicBlockData::new_stmts( + vec![Statement::new(source_info, StatementKind::StorageDead(local))], + Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), + false, + )); + let kind = if needs_drop { + TerminatorKind::Drop { + place, + target: drop_one, + unwind, + replace, + drop: dropline, + async_fut, + } + } else { + TerminatorKind::Goto { target: drop_one } + }; + drop_one = patch.new_block(BasicBlockData::new_stmts( + vec![], + Some(Terminator { source_info, kind }), + false, + )); + target = drop_one; + unwind = UnwindAction::Cleanup(unwind_one); + cleanup = unwind_one; + } + } + patch.patch_terminator(block, TerminatorKind::Goto { target }); +} + +fn rewrite_drop_coroutine_struct<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut blocks = DenseBitSet::new_empty(body.basic_blocks.len()); + for (block, block_data) in body.basic_blocks.iter_enumerated() { + let Terminator { source_info: _, kind: TerminatorKind::Drop { place, .. } } = + block_data.terminator() + else { + continue; + }; + let Some(local) = place.as_local() else { continue }; + if local == ty::CAPTURE_STRUCT_LOCAL { + blocks.insert(block); + } + } + let mut patch = MirPatch::new(body); + for block in blocks.iter() { + rewrite_one_drop_coroutine_struct(tcx, &mut patch, body, block, substitution_mapping); + } + patch.apply(body); +} + +fn insert_substitution_prologue<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut patch = MirPatch::new(body); + let mut stmts = Vec::with_capacity(2 * substitution_mapping.len()); + for &UpvarSubstitution { local, reloc, upvar_place, span, name: _ } in substitution_mapping { + // For each upvar-local _$i + let source_info = SourceInfo::outermost(span); + // StorageLive(_$i) + stmts.push(Statement::new(source_info, StatementKind::StorageLive(local))); + // Use a fresh local _$i' here, so as to avoid potential field permutation + // StorageLive(_$i') + stmts.push(Statement::new(source_info, StatementKind::StorageLive(reloc))); + // _$i' = move $ + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + reloc.into(), + mir::Rvalue::Use(mir::Operand::Move(upvar_place)), + ))), + )); + } + for &UpvarSubstitution { local, reloc, upvar_place: _, span, name: _ } in substitution_mapping { + let source_info = SourceInfo::outermost(span); + // _$i = move $i' + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + local.into(), + mir::Rvalue::Use(mir::Operand::Move(reloc.into())), + ))), + )); + stmts.push(Statement::new(source_info, StatementKind::StorageDead(reloc))); + } + let source_info = SourceInfo::outermost(body.span); + let prologue = patch.new_block(BasicBlockData::new_stmts( + stmts, + Some(Terminator { source_info, kind: TerminatorKind::Goto { target: START_BLOCK } }), + false, + )); + patch.apply(body); + + // Manually patch so that prologue is the new entry-point + let preds = body.basic_blocks.predecessors()[START_BLOCK].clone(); + let basic_blocks = body.basic_blocks.as_mut(); + for pred in preds { + basic_blocks[pred].terminator_mut().successors_mut(|target| { + if *target == START_BLOCK { + *target = prologue; + } + }); + } + basic_blocks.swap(START_BLOCK, prologue); +} + +/// Occasionally there are upvar locals left without `StorageDead` because +/// they do not have destructors. +/// We need to mark them daed for correctness, as previously the entire +/// capture structure was marked dead and now we need to mark them one at +/// a time. +fn patch_missing_storage_deads<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let always_live_locals = &always_storage_live_locals(body); + + let mut maybe_storage_live = MaybeStorageLive::new(Cow::Borrowed(always_live_locals)) + .iterate_to_fixpoint(tcx, body, None) + .into_results_cursor(body); + + let mut upvar_locals = DenseBitSet::new_empty(body.local_decls.len()); + for subst in substitution_mapping { + upvar_locals.insert(subst.local); + } + let mut storage_dead_stmts: IndexVec> = + IndexVec::from_elem_n(smallvec![], body.basic_blocks.len()); + for (block, data) in body.basic_blocks.iter_enumerated() { + if !data.is_cleanup && matches!(data.terminator().kind, TerminatorKind::Return) { + let nr_stmts = data.statements.len(); + maybe_storage_live + .seek_after_primary_effect(mir::Location { block, statement_index: nr_stmts }); + let mut missing_locals = maybe_storage_live.get().clone(); + missing_locals.intersect(&upvar_locals); + let source_info = data.terminator().source_info; + for local in missing_locals.iter() { + storage_dead_stmts[block] + .push(mir::Statement::new(source_info, mir::StatementKind::StorageDead(local))); + } + } + } + let basic_blocks = body.basic_blocks.as_mut(); + for (block, storage_deaths) in storage_dead_stmts.iter_enumerated_mut() { + basic_blocks[block].statements.extend(storage_deaths.drain(..)); + } +} + +fn hydrate_var_debug_info<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + for subst in substitution_mapping { + body.var_debug_info.push(mir::VarDebugInfo { + name: subst.name, + source_info: SourceInfo::outermost(subst.span), + composite: None, + value: mir::VarDebugInfoContents::Place(Place::from(subst.local)), + argument_index: None, + }); + } +} diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index bc914ea656415..00d81cc3b3ccd 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -40,6 +40,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> { ty, self.local_decls[p_ref.local].source_info.span, LocalInfo::DerefTemp, + false, ); // We are adding current p_ref's projections to our diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 9ff7e0b550030..ce5dec246da81 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -127,7 +127,7 @@ declare_passes! { pub mod cleanup_post_borrowck : CleanupPostBorrowck; mod copy_prop : CopyProp; - mod coroutine : StateTransform; + mod coroutine : RelocateUpvars, StateTransform; mod coverage : InstrumentCoverage; mod ctfe_limit : CtfeLimit; mod dataflow_const_prop : DataflowConstProp; diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index ab09cdf787ee7..987510f714b53 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -311,12 +311,21 @@ fn run_passes_inner<'tcx>( if let Some(dumper) = dumper { dumper.set_disambiguator(&"after").dump_mir(body); } + let (dialect, phase_num) = body.phase.index(); if validate { - validate_body(tcx, body, format!("after pass {pass_name}")); + validate_body( + tcx, + body, + format!("after pass {pass_name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } if lint { - lint_body(tcx, body, format!("after pass {pass_name}")); + lint_body( + tcx, + body, + format!("after pass {pass_name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } body.pass_count += 1; diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index 2c535d011a0e2..7baf85b82e6d6 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -175,10 +175,14 @@ impl<'tcx> MirPatch<'tcx> { ty: Ty<'tcx>, span: Span, local_info: LocalInfo<'tcx>, + immutable: bool, ) -> Local { let index = self.next_local; self.next_local += 1; let mut new_decl = LocalDecl::new(ty, span); + if immutable { + new_decl = new_decl.immutable(); + } **new_decl.local_info.as_mut().unwrap_crate_local() = local_info; self.new_locals.push(new_decl); Local::new(index) diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index bca8ffb693b90..524556a4e42f1 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -48,7 +48,7 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { _context: PlaceContext, _location: Location, ) { - if place.local == Local::from_u32(1) { + if place.local == ty::CAPTURE_STRUCT_LOCAL { if place.projection.len() == 1 { assert!(matches!( place.projection.first(), @@ -65,9 +65,8 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { } } +#[instrument(level = "debug", skip(tcx))] fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> { - debug!("make_shim({:?})", instance); - let mut result = match instance { ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance), ty::InstanceKind::VTableShim(def_id) => { @@ -194,6 +193,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< tcx, &mut body, &[ + &crate::coroutine::RelocateUpvars::new(true), // We must always relocate to ensure valid MIR &mentioned_items::MentionedItems, &abort_unwinding_calls::AbortUnwindingCalls, &add_call_guards::CriticalCallEdges, diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index 18d09473c191e..dec56456b8cac 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -11,6 +11,7 @@ use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt, TypeVisitableExt}; use super::*; use crate::patch::MirPatch; +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_destructor_ctor_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, @@ -39,12 +40,12 @@ pub(super) fn build_async_destructor_ctor_shim<'tcx>( } // build_drop_shim analog for async drop glue (for generated coroutine poll function) +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_drop_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ty: Ty<'tcx>, ) -> Body<'tcx> { - debug!("build_async_drop_shim(def_id={:?}, ty={:?})", def_id, ty); let ty::Coroutine(_, parent_args) = ty.kind() else { bug!(); }; @@ -193,6 +194,7 @@ pub(super) fn build_future_drop_poll_shim<'tcx>( // `async_drop_in_place::{closure}.poll()` is converted into `T.future_drop_poll()`. // Every coroutine has its `poll` (calculate yourself a little further) // and its `future_drop_poll` (drop yourself a little further). +#[instrument(level = "debug", skip_all, fields(span))] fn build_adrop_for_coroutine_shim<'tcx>( tcx: TyCtxt<'tcx>, proxy_ty: Ty<'tcx>, @@ -200,33 +202,43 @@ fn build_adrop_for_coroutine_shim<'tcx>( span: Span, instance: ty::InstanceKind<'tcx>, ) -> Body<'tcx> { - let ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { + debug!("proxy_ty={proxy_ty:#?}"); + debug!("impl_ty={impl_ty:#?}"); + debug!("instance={instance:#?}"); + let &ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { bug!("build_adrop_for_coroutine_shim not for coroutine impl type: ({:?})", instance); }; let proxy_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, proxy_ty); - // taking _1.0 (impl from Pin) - let pin_proxy_layout_local = Local::new(1); - let source_info = SourceInfo::outermost(span); - // converting `(_1: Pin<&mut CorLayout>, _2: &mut Context<'_>) -> Poll<()>` - // into `(_1: Pin<&mut ProxyLayout>, _2: &mut Context<'_>) -> Poll<()>` - // let mut _x: &mut CorLayout = &*_1.0.0; - // Replace old _1.0 accesses into _x accesses; - let body = tcx.optimized_mir(*coroutine_def_id).future_drop_poll().unwrap(); + let body = tcx.optimized_mir(coroutine_def_id).future_drop_poll().unwrap(); let mut body: Body<'tcx> = EarlyBinder::bind(body.clone()).instantiate(tcx, impl_args); + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_coroutine_shim_before", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } + debug!(?body.source.instance, "before"); body.source.instance = instance; body.phase = MirPhase::Runtime(RuntimePhase::Initial); body.var_debug_info.clear(); + // Here we convert `(_1: Pin<&mut InnerCoroutine>, _2: &mut Context<'_>) -> Poll<()>` + // into `(_1: Pin<&mut ProxyCoroutine>, _2: &mut Context<'_>) -> Poll<()>`. + // **The Schematic** + // We make a new local `_x`: + // let mut _x: &mut InnerCoroutine = &* ((*_1.0) as variant#0).0; + // Then replace projections on `_1.0` into projections on `_x`. + let pin_proxy_layout_local = Local::new(1); + let source_info = SourceInfo::outermost(span); let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span)); let args = tcx.mk_args(&[proxy_ref.into()]); let pin_proxy_ref = Ty::new_adt(tcx, pin_adt_ref, args); + // This is the type of the vanilla `InnerCoroutine` let cor_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, impl_ty); let proxy_ref_local = body.local_decls.push(LocalDecl::new(proxy_ref, span)); let cor_ref_local = body.local_decls.push(LocalDecl::new(cor_ref, span)); + // Batch replacement of `_1.0` with `_x` FixProxyFutureDropVisitor { tcx, replace_to: cor_ref_local }.visit_body(&mut body); - // Now changing first arg from Pin<&mut ImplCoroutine> to Pin<&mut ProxyCoroutine> + // Now changing first arg from Pin<&mut InnerCoroutine> to Pin<&mut ProxyCoroutine> body.local_decls[pin_proxy_layout_local] = LocalDecl::new(pin_proxy_ref, span); { @@ -250,11 +262,15 @@ fn build_adrop_for_coroutine_shim<'tcx>( if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( - &[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], tcx, ); cor_ptr_local = body.local_decls.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) body.basic_blocks_mut()[START_BLOCK].statements.insert( idx, Statement::new( @@ -283,6 +299,9 @@ fn build_adrop_for_coroutine_shim<'tcx>( ), ); } + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_coroutine_shim", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } body } @@ -342,10 +361,16 @@ fn build_adrop_for_adrop_shim<'tcx>( proxy_ty.find_async_drop_impl_coroutine(tcx, |ty| { if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); - let impl_ptr_place = Place::from(cor_ptr_local) - .project_deeper(&[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], tcx); + let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], + tcx, + ); cor_ptr_local = locals.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) statements.push(Statement::new( source_info, StatementKind::Assign(Box::new(( @@ -421,5 +446,8 @@ fn build_adrop_for_adrop_shim<'tcx>( let source = MirSource::from_instance(instance); let mut body = new_body(source, blocks, locals, sig.inputs().len(), span); body.phase = MirPhase::Runtime(RuntimePhase::Initial); - return body; + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_adrop_shim", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } + body } diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 8b5efb7420582..19bb0f078962a 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -451,7 +451,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyLocals { // Only bother running the `LocalUpdater` if we actually found locals to remove. if map.iter().any(Option::is_none) { // Update references to all vars and tmps now - let mut updater = LocalUpdater { map, tcx }; + let mut updater = LocalUpdater { map: &map, tcx }; updater.visit_body_preserves_cfg(body); body.local_decls.shrink_to_fit(); @@ -647,12 +647,12 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod } } -struct LocalUpdater<'tcx> { - map: IndexVec>, +struct LocalUpdater<'tcx, 'a> { + map: &'a IndexSlice>, tcx: TyCtxt<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { +impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx, '_> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 95873484b6529..8cc59494251ab 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -14,7 +14,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ - self, CoroutineArgsExt, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, + self, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, }; use rustc_middle::{bug, span_bug}; use rustc_trait_selection::traits::ObligationCtxt; @@ -797,14 +797,11 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { }; ty::EarlyBinder::bind(f_ty.ty).instantiate(self.tcx, args) - } else { - let Some(&f_ty) = args.as_coroutine().prefix_tys().get(f.index()) - else { - fail_out_of_bounds(self, location); - return; - }; - + } else if let Some(&f_ty) = args.as_coroutine().upvar_tys().get(f.index()) { f_ty + } else { + fail_out_of_bounds(self, location); + return; }; check_equal(self, location, f_ty); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index d1426ff55fbd6..d6d39f7639f59 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -3243,9 +3243,9 @@ pub(crate) mod dep_tracking { CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OomStrategy, OptLevel, OutFileName, - OutputType, OutputTypes, PatchableFunctionEntry, Polonius, RemapPathScopeComponents, - ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, - SymbolManglingVersion, WasiExecModel, + OutputType, OutputTypes, PackCoroutineLayout, PatchableFunctionEntry, Polonius, + RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, + SwitchWithOptPath, SymbolManglingVersion, WasiExecModel, }; use crate::lint; use crate::utils::NativeLib; @@ -3348,6 +3348,7 @@ pub(crate) mod dep_tracking { Polonius, InliningThreshold, FunctionReturn, + PackCoroutineLayout, Align, ); @@ -3602,6 +3603,30 @@ pub enum FunctionReturn { ThunkExtern, } +/// Layout optimisation for Coroutines +#[derive( + Clone, + Copy, + PartialEq, + Eq, + Hash, + HashStable_Generic, + Debug, + Default, + Decodable, + Encodable +)] +pub enum PackCoroutineLayout { + /// Keep coroutine captured variables throughout all states + #[default] + No, + + /// Allow coroutine captured variables that are used only once + /// before the first suspension to be freed up for storage + /// in all other suspension states + CapturesOnly, +} + /// Whether extra span comments are included when dumping MIR, via the `-Z mir-include-spans` flag. /// By default, only enabled in the NLL MIR dumps, and disabled in all other passes. #[derive(Clone, Copy, Default, PartialEq, Debug)] diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 6dd90546de1b0..63c2604f341ee 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -805,6 +805,7 @@ mod desc { pub(crate) const parse_panic_strategy: &str = "either `unwind`, `abort`, or `immediate-abort`"; pub(crate) const parse_on_broken_pipe: &str = "either `kill`, `error`, or `inherit`"; pub(crate) const parse_patchable_function_entry: &str = "either two comma separated integers (total_nops,prefix_nops), with prefix_nops <= total_nops, or one integer (total_nops)"; + pub(crate) const parse_pack_coroutine_layout: &str = "either `no` or `captures-only`"; pub(crate) const parse_opt_panic_strategy: &str = parse_panic_strategy; pub(crate) const parse_oom_strategy: &str = "either `panic` or `abort`"; pub(crate) const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; @@ -1951,6 +1952,18 @@ pub mod parse { true } + pub(crate) fn parse_pack_coroutine_layout( + slot: &mut PackCoroutineLayout, + v: Option<&str>, + ) -> bool { + *slot = match v { + Some("no") => PackCoroutineLayout::No, + Some("captures-only") => PackCoroutineLayout::CapturesOnly, + _ => return false, + }; + true + } + pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool { match v { Some("always" | "yes") => { @@ -2504,6 +2517,8 @@ options! { "panic strategy for out-of-memory handling"), osx_rpath_install_name: bool = (false, parse_bool, [TRACKED], "pass `-install_name @rpath/...` to the macOS linker (default: no)"), + pack_coroutine_layout: PackCoroutineLayout = (PackCoroutineLayout::default(), parse_pack_coroutine_layout, [TRACKED], + "set strategy to pack coroutine state layout (default: no)"), packed_bundled_libs: bool = (false, parse_bool, [TRACKED], "change rlib format to store native libraries as archives"), panic_abort_tests: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs index 136a61598d50e..82178330ad3e2 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs @@ -6,7 +6,7 @@ use std::iter; use std::path::PathBuf; use itertools::{EitherOrBoth, Itertools}; -use rustc_abi::ExternAbi; +use rustc_abi::{ExternAbi, FIRST_VARIANT}; use rustc_data_structures::fx::FxHashSet; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_errors::codes::*; @@ -2492,18 +2492,28 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { && let Some(coroutine_info) = self.tcx.mir_coroutine_witnesses(coroutine_did) { debug!(?coroutine_info); - 'find_source: for (variant, source_info) in - coroutine_info.variant_fields.iter().zip(&coroutine_info.variant_source_info) + // Variants are scanned "in reverse" because suspension points + // tend to contain more diagnostic information than the unresumed state. + 'find_source: for ((variant_idx, variant), source_info) in coroutine_info + .variant_fields + .iter_enumerated() + .zip(&coroutine_info.variant_source_info) + .rev() { debug!(?variant); for &local in variant { let decl = &coroutine_info.field_tys[local]; debug!(?decl); if ty_matches(ty::Binder::dummy(decl.ty)) && !decl.ignore_for_traits { - interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( - decl.source_info.span, - Some((source_info.span, from_awaited_ty)), - )); + let span = decl.source_info.span; + if variant_idx == FIRST_VARIANT { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Upvar(span)); + } else { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( + span, + Some((source_info.span, from_awaited_ty)), + )); + } break 'find_source; } } diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs index 2e60805cd10a5..8292f312cbcc2 100644 --- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs +++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs @@ -1,6 +1,8 @@ +use rustc_abi::VariantIdx; use rustc_data_structures::fx::FxHashSet; use rustc_infer::traits::query::type_op::DropckOutlives; use rustc_middle::traits::query::{DropckConstraint, DropckOutlivesResult}; +use rustc_middle::ty::data_structures::IndexSet; use rustc_middle::ty::{self, EarlyBinder, ParamEnvAnd, Ty, TyCtxt}; use rustc_span::Span; use tracing::{debug, instrument}; @@ -354,7 +356,14 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>( // FIXME(@lcnr): Why do we erase regions in the env here? Seems odd let typing_env = tcx.erase_and_anonymize_regions(typing_env); let needs_drop = tcx.mir_coroutine_witnesses(def_id).is_some_and(|witness| { - witness.field_tys.iter().any(|field| field.ty.needs_drop(tcx, typing_env)) + let upvar_locals: IndexSet<_> = + witness.variant_fields[VariantIdx::ZERO].iter().copied().collect(); + // As a reminder, upvars also appear in the UNRESUMED state-variant + witness + .field_tys + .iter_enumerated() + .filter_map(|(corsl, ty)| (!upvar_locals.contains(&corsl)).then_some(ty)) + .any(|field| field.ty.needs_drop(tcx, typing_env)) }); if needs_drop { // Pushing types directly to `constraints.outlives` is equivalent diff --git a/compiler/rustc_traits/src/coroutine_witnesses.rs b/compiler/rustc_traits/src/coroutine_witnesses.rs index 2544cd8a13cd8..5505d1646ee06 100644 --- a/compiler/rustc_traits/src/coroutine_witnesses.rs +++ b/compiler/rustc_traits/src/coroutine_witnesses.rs @@ -2,24 +2,40 @@ use rustc_infer::infer::TyCtxtInferExt; use rustc_infer::infer::canonical::query_response::make_query_region_constraints; use rustc_infer::infer::resolve::OpportunisticRegionResolver; use rustc_infer::traits::{Obligation, ObligationCause}; +use rustc_middle::mir::CoroutineLayout; +use rustc_middle::ty::data_structures::IndexSet; use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, fold_regions}; use rustc_span::def_id::DefId; use rustc_trait_selection::traits::{ObligationCtxt, with_replaced_escaping_bound_vars}; +use tracing::instrument; /// Return the set of types that should be taken into account when checking /// trait bounds on a coroutine's internal state. This properly replaces /// `ReErased` with new existential bound lifetimes. +#[instrument(level = "debug", skip(tcx), ret)] pub(crate) fn coroutine_hidden_types<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes>>> { - let coroutine_layout = tcx.mir_coroutine_witnesses(def_id); + let Some(coroutine_layout) = tcx.mir_coroutine_witnesses(def_id) else { + return ty::EarlyBinder::bind(ty::Binder::dummy(ty::CoroutineWitnessTypes { + types: ty::List::empty(), + assumptions: ty::List::empty(), + })); + }; + let mut internal_fields: IndexSet<_> = + coroutine_layout.variant_fields.iter().skip(1).flat_map(|f| f.iter().copied()).collect(); + for upvar in &coroutine_layout.variant_fields[CoroutineLayout::UNRESUMED] { + internal_fields.swap_remove(upvar); + } let mut vars = vec![]; let bound_tys = tcx.mk_type_list_from_iter( - coroutine_layout - .as_ref() - .map_or_else(|| [].iter(), |l| l.field_tys.iter()) - .filter(|decl| !decl.ignore_for_traits) + internal_fields + .into_iter() + .filter_map(|saved_local| { + let saved_ty = &coroutine_layout.field_tys[saved_local]; + (!saved_ty.ignore_for_traits).then_some(saved_ty) + }) .map(|decl| { let ty = fold_regions(tcx, decl.ty, |re, debruijn| { assert_eq!(re, tcx.lifetimes.re_erased); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 317d101dafe05..9926cede48c8a 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -11,6 +11,7 @@ use rustc_hir::attrs::AttributeKind; use rustc_hir::find_attr; use rustc_index::IndexVec; use rustc_middle::bug; +use rustc_middle::mir::CoroutineSavedLocal; use rustc_middle::query::Providers; use rustc_middle::traits::ObligationCause; use rustc_middle::ty::layout::{ @@ -20,6 +21,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ self, AdtDef, CoroutineArgsExt, EarlyBinder, PseudoCanonicalInput, Ty, TyCtxt, TypeVisitableExt, }; +use rustc_session::config::PackCoroutineLayout; use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo}; use rustc_span::{Symbol, sym}; use tracing::{debug, instrument}; @@ -175,6 +177,7 @@ fn extract_const_value<'tcx>( } } +#[instrument(level = "debug", skip(cx), ret)] fn layout_of_uncached<'tcx>( cx: &LayoutCx<'tcx>, ty: Ty<'tcx>, @@ -490,7 +493,7 @@ fn layout_of_uncached<'tcx>( let info = tcx.coroutine_layout(def_id, args)?; - let local_layouts = info + let local_layouts: IndexVec<_, _> = info .field_tys .iter() .map(|local| { @@ -498,22 +501,31 @@ fn layout_of_uncached<'tcx>( let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty.instantiate(tcx, args)); cx.spanned_layout_of(uninit_ty, local.source_info.span) }) - .try_collect::>()?; + .try_collect()?; - let prefix_layouts = args + let relocated_upvars = IndexVec::from_fn_n( + |local: CoroutineSavedLocal| info.relocated_upvars.get(&local).copied(), + info.field_tys.len(), + ); + let pack = match info.pack { + PackCoroutineLayout::No => rustc_abi::PackCoroutineLayout::Classic, + PackCoroutineLayout::CapturesOnly => rustc_abi::PackCoroutineLayout::CapturesOnly, + }; + let upvar_layouts = args .as_coroutine() - .prefix_tys() + .upvar_tys() .iter() .map(|ty| cx.layout_of(ty)) - .try_collect::>()?; - + .collect::>()?; let layout = cx .calc .coroutine( &local_layouts, - prefix_layouts, + &relocated_upvars, + upvar_layouts, &info.variant_fields, &info.storage_conflicts, + pack, |tag| TyAndLayout { ty: tag.primitive().to_ty(tcx), layout: tcx.mk_layout(LayoutData::scalar(cx, tag)), @@ -912,7 +924,11 @@ fn variant_info_for_coroutine<'tcx>( .then(|| Symbol::intern(&field_layout.ty.to_string())), } }) - .chain(upvar_fields.iter().copied()) + .chain( + if variant_idx == FIRST_VARIANT { &upvar_fields[..] } else { &[] } + .iter() + .copied(), + ) .collect(); // If the variant has no state-specific fields, then it's the size of the upvars. diff --git a/src/tools/clippy/tests/ui/future_not_send.stderr b/src/tools/clippy/tests/ui/future_not_send.stderr index e366dc2d21958..8521eb13f62d7 100644 --- a/src/tools/clippy/tests/ui/future_not_send.stderr +++ b/src/tools/clippy/tests/ui/future_not_send.stderr @@ -93,10 +93,10 @@ LL | pub async fn public_future(&self) { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future returned by `public_future` is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> tests/ui/future_not_send.rs:49:32 + --> tests/ui/future_not_send.rs:49:33 | LL | pub async fn public_future(&self) { - | ^^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` + | ^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` = note: `std::rc::Rc<[u8]>` doesn't implement `std::marker::Sync` error: future cannot be sent between threads safely diff --git a/src/tools/clippy/tests/ui/large_futures.stderr b/src/tools/clippy/tests/ui/large_futures.stderr index 4280c9e2af284..06956ef9d4a88 100644 --- a/src/tools/clippy/tests/ui/large_futures.stderr +++ b/src/tools/clippy/tests/ui/large_futures.stderr @@ -1,4 +1,4 @@ -error: large future with a size of 16385 bytes +error: large future with a size of 32769 bytes --> tests/ui/large_futures.rs:13:9 | LL | big_fut([0u8; 1024 * 16]).await; @@ -7,19 +7,19 @@ LL | big_fut([0u8; 1024 * 16]).await; = note: `-D clippy::large-futures` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(clippy::large_futures)]` -error: large future with a size of 16386 bytes +error: large future with a size of 32770 bytes --> tests/ui/large_futures.rs:16:5 | LL | f.await | ^ help: consider `Box::pin` on it: `Box::pin(f)` -error: large future with a size of 16387 bytes +error: large future with a size of 32771 bytes --> tests/ui/large_futures.rs:21:9 | LL | wait().await; | ^^^^^^ help: consider `Box::pin` on it: `Box::pin(wait())` -error: large future with a size of 16387 bytes +error: large future with a size of 32771 bytes --> tests/ui/large_futures.rs:27:13 | LL | wait().await; @@ -31,7 +31,7 @@ error: large future with a size of 65540 bytes LL | foo().await; | ^^^^^ help: consider `Box::pin` on it: `Box::pin(foo())` -error: large future with a size of 49159 bytes +error: large future with a size of 98311 bytes --> tests/ui/large_futures.rs:38:5 | LL | calls_fut(fut).await; diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.fixed b/src/tools/clippy/tests/ui/needless_lifetimes.fixed index 15ca409c95bd1..ef8a4a91677d0 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.fixed +++ b/src/tools/clippy/tests/ui/needless_lifetimes.fixed @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.rs b/src/tools/clippy/tests/ui/needless_lifetimes.rs index af9649d729872..86da853dc238f 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.rs +++ b/src/tools/clippy/tests/ui/needless_lifetimes.rs @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.stderr b/src/tools/clippy/tests/ui/needless_lifetimes.stderr index 138d0498c43e4..c7669e4d175b4 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.stderr +++ b/src/tools/clippy/tests/ui/needless_lifetimes.stderr @@ -1,5 +1,5 @@ error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:19:23 + --> tests/ui/needless_lifetimes.rs:20:23 | LL | fn distinct_lifetimes<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: u8) {} | ^^ ^^ ^^ ^^ @@ -13,7 +13,7 @@ LL + fn distinct_lifetimes(_x: &u8, _y: &u8, _z: u8) {} | error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:22:24 + --> tests/ui/needless_lifetimes.rs:23:24 | LL | fn distinct_and_static<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: &'static u8) {} | ^^ ^^ ^^ ^^ @@ -25,7 +25,7 @@ LL + fn distinct_and_static(_x: &u8, _y: &u8, _z: &'static u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:33:15 + --> tests/ui/needless_lifetimes.rs:34:15 | LL | fn in_and_out<'a>(x: &'a u8, _y: u8) -> &'a u8 { | ^^ ^^ ^^ @@ -37,7 +37,7 @@ LL + fn in_and_out(x: &u8, _y: u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:46:31 + --> tests/ui/needless_lifetimes.rs:47:31 | LL | fn multiple_in_and_out_2a<'a, 'b>(x: &'a u8, _y: &'b u8) -> &'a u8 { | ^^ ^^ @@ -49,7 +49,7 @@ LL + fn multiple_in_and_out_2a<'a>(x: &'a u8, _y: &u8) -> &'a u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:54:27 + --> tests/ui/needless_lifetimes.rs:55:27 | LL | fn multiple_in_and_out_2b<'a, 'b>(_x: &'a u8, y: &'b u8) -> &'b u8 { | ^^ ^^ @@ -61,7 +61,7 @@ LL + fn multiple_in_and_out_2b<'b>(_x: &u8, y: &'b u8) -> &'b u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:72:26 + --> tests/ui/needless_lifetimes.rs:73:26 | LL | fn deep_reference_1a<'a, 'b>(x: &'a u8, _y: &'b u8) -> Result<&'a u8, ()> { | ^^ ^^ @@ -73,7 +73,7 @@ LL + fn deep_reference_1a<'a>(x: &'a u8, _y: &u8) -> Result<&'a u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:80:22 + --> tests/ui/needless_lifetimes.rs:81:22 | LL | fn deep_reference_1b<'a, 'b>(_x: &'a u8, y: &'b u8) -> Result<&'b u8, ()> { | ^^ ^^ @@ -85,7 +85,7 @@ LL + fn deep_reference_1b<'b>(_x: &u8, y: &'b u8) -> Result<&'b u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:90:21 + --> tests/ui/needless_lifetimes.rs:91:21 | LL | fn deep_reference_3<'a>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> { | ^^ ^^ ^^ @@ -97,7 +97,7 @@ LL + fn deep_reference_3(x: &u8, _y: u8) -> Result<&u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:96:28 + --> tests/ui/needless_lifetimes.rs:97:28 | LL | fn where_clause_without_lt<'a, T>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> | ^^ ^^ ^^ @@ -109,7 +109,7 @@ LL + fn where_clause_without_lt(x: &u8, _y: u8) -> Result<&u8, ()> | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:127:21 + --> tests/ui/needless_lifetimes.rs:128:21 | LL | fn self_and_out<'s>(&'s self) -> &'s u8 { | ^^ ^^ ^^ @@ -121,7 +121,7 @@ LL + fn self_and_out(&self) -> &u8 { | error: the following explicit lifetimes could be elided: 't - --> tests/ui/needless_lifetimes.rs:135:30 + --> tests/ui/needless_lifetimes.rs:136:30 | LL | fn self_and_in_out_1<'s, 't>(&'s self, _x: &'t u8) -> &'s u8 { | ^^ ^^ @@ -133,7 +133,7 @@ LL + fn self_and_in_out_1<'s>(&'s self, _x: &u8) -> &'s u8 { | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:143:26 + --> tests/ui/needless_lifetimes.rs:144:26 | LL | fn self_and_in_out_2<'s, 't>(&'s self, x: &'t u8) -> &'t u8 { | ^^ ^^ @@ -145,7 +145,7 @@ LL + fn self_and_in_out_2<'t>(&self, x: &'t u8) -> &'t u8 { | error: the following explicit lifetimes could be elided: 's, 't - --> tests/ui/needless_lifetimes.rs:148:29 + --> tests/ui/needless_lifetimes.rs:149:29 | LL | fn distinct_self_and_in<'s, 't>(&'s self, _x: &'t u8) {} | ^^ ^^ ^^ ^^ @@ -157,7 +157,7 @@ LL + fn distinct_self_and_in(&self, _x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:172:21 + --> tests/ui/needless_lifetimes.rs:173:21 | LL | fn struct_with_lt4b<'a, 'b>(_foo: &'a Foo<'b>) -> &'b str { | ^^ ^^ @@ -169,7 +169,7 @@ LL + fn struct_with_lt4b<'b>(_foo: &Foo<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:188:22 + --> tests/ui/needless_lifetimes.rs:189:22 | LL | fn trait_obj_elided2<'a>(_arg: &'a dyn Drop) -> &'a str { | ^^ ^^ ^^ @@ -181,7 +181,7 @@ LL + fn trait_obj_elided2(_arg: &dyn Drop) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:199:20 + --> tests/ui/needless_lifetimes.rs:200:20 | LL | fn alias_with_lt4b<'a, 'b>(_foo: &'a FooAlias<'b>) -> &'b str { | ^^ ^^ @@ -193,7 +193,7 @@ LL + fn alias_with_lt4b<'b>(_foo: &FooAlias<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:204:30 + --> tests/ui/needless_lifetimes.rs:205:30 | LL | fn named_input_elided_output<'a>(_arg: &'a str) -> &str { | ^^ ^^ ^ @@ -205,7 +205,7 @@ LL + fn named_input_elided_output(_arg: &str) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:213:19 + --> tests/ui/needless_lifetimes.rs:214:19 | LL | fn trait_bound_ok<'a, T: WithLifetime<'static>>(_: &'a u8, _: T) { | ^^ ^^ @@ -217,7 +217,7 @@ LL + fn trait_bound_ok>(_: &u8, _: T) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:249:24 + --> tests/ui/needless_lifetimes.rs:250:24 | LL | fn needless_lt<'a>(x: &'a u8) {} | ^^ ^^ @@ -229,7 +229,7 @@ LL + fn needless_lt(x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:254:24 + --> tests/ui/needless_lifetimes.rs:255:24 | LL | fn needless_lt<'a>(_x: &'a u8) {} | ^^ ^^ @@ -241,7 +241,7 @@ LL + fn needless_lt(_x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:285:55 + --> tests/ui/needless_lifetimes.rs:286:55 | LL | fn impl_trait_elidable_nested_anonymous_lifetimes<'a>(i: &'a i32, f: impl Fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -253,7 +253,7 @@ LL + fn impl_trait_elidable_nested_anonymous_lifetimes(i: &i32, f: impl Fn(& | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:295:26 + --> tests/ui/needless_lifetimes.rs:296:26 | LL | fn generics_elidable<'a, T: Fn(&i32) -> &i32>(i: &'a i32, f: T) -> &'a i32 { | ^^ ^^ ^^ @@ -265,7 +265,7 @@ LL + fn generics_elidable &i32>(i: &i32, f: T) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:308:30 + --> tests/ui/needless_lifetimes.rs:309:30 | LL | fn where_clause_elidable<'a, T>(i: &'a i32, f: T) -> &'a i32 | ^^ ^^ ^^ @@ -277,7 +277,7 @@ LL + fn where_clause_elidable(i: &i32, f: T) -> &i32 | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:324:28 + --> tests/ui/needless_lifetimes.rs:325:28 | LL | fn pointer_fn_elidable<'a>(i: &'a i32, f: fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -289,7 +289,7 @@ LL + fn pointer_fn_elidable(i: &i32, f: fn(&i32) -> &i32) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:338:28 + --> tests/ui/needless_lifetimes.rs:339:28 | LL | fn nested_fn_pointer_3<'a>(_: &'a i32) -> fn(fn(&i32) -> &i32) -> i32 { | ^^ ^^ @@ -301,7 +301,7 @@ LL + fn nested_fn_pointer_3(_: &i32) -> fn(fn(&i32) -> &i32) -> i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:342:28 + --> tests/ui/needless_lifetimes.rs:343:28 | LL | fn nested_fn_pointer_4<'a>(_: &'a i32) -> impl Fn(fn(&i32)) { | ^^ ^^ @@ -313,7 +313,7 @@ LL + fn nested_fn_pointer_4(_: &i32) -> impl Fn(fn(&i32)) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:365:21 + --> tests/ui/needless_lifetimes.rs:366:21 | LL | fn implicit<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -325,7 +325,7 @@ LL + fn implicit(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:369:25 + --> tests/ui/needless_lifetimes.rs:370:25 | LL | fn implicit_mut<'a>(&'a mut self) -> &'a () { | ^^ ^^ ^^ @@ -337,7 +337,7 @@ LL + fn implicit_mut(&mut self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:374:21 + --> tests/ui/needless_lifetimes.rs:375:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -349,7 +349,7 @@ LL + fn explicit(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:379:25 + --> tests/ui/needless_lifetimes.rs:380:25 | LL | fn explicit_mut<'a>(self: &'a mut Rc) -> &'a () { | ^^ ^^ ^^ @@ -361,7 +361,7 @@ LL + fn explicit_mut(self: &mut Rc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:392:31 + --> tests/ui/needless_lifetimes.rs:393:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -373,7 +373,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:399:21 + --> tests/ui/needless_lifetimes.rs:400:21 | LL | fn implicit<'a>(&'a self) -> &'a (); | ^^ ^^ ^^ @@ -385,7 +385,7 @@ LL + fn implicit(&self) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:401:30 + --> tests/ui/needless_lifetimes.rs:402:30 | LL | fn implicit_provided<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -397,7 +397,7 @@ LL + fn implicit_provided(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:407:21 + --> tests/ui/needless_lifetimes.rs:408:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a (); | ^^ ^^ ^^ @@ -409,7 +409,7 @@ LL + fn explicit(self: &Arc) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:410:30 + --> tests/ui/needless_lifetimes.rs:411:30 | LL | fn explicit_provided<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -421,7 +421,7 @@ LL + fn explicit_provided(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:421:31 + --> tests/ui/needless_lifetimes.rs:422:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a (); | ^^ ^^ ^^ @@ -433,7 +433,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:423:40 + --> tests/ui/needless_lifetimes.rs:424:40 | LL | fn lifetime_elsewhere_provided<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -445,7 +445,7 @@ LL + fn lifetime_elsewhere_provided(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:433:12 + --> tests/ui/needless_lifetimes.rs:434:12 | LL | fn foo<'a>(x: &'a u8, y: &'_ u8) {} | ^^ ^^ @@ -457,7 +457,7 @@ LL + fn foo(x: &u8, y: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:436:12 + --> tests/ui/needless_lifetimes.rs:437:12 | LL | fn bar<'a>(x: &'a u8, y: &'_ u8, z: &'_ u8) {} | ^^ ^^ @@ -469,7 +469,7 @@ LL + fn bar(x: &u8, y: &'_ u8, z: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:444:18 + --> tests/ui/needless_lifetimes.rs:445:18 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ @@ -481,7 +481,7 @@ LL + fn one_input(x: &u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:450:42 + --> tests/ui/needless_lifetimes.rs:451:42 | LL | fn multiple_inputs_output_not_elided<'a, 'b>(x: &'a u8, y: &'b u8, z: &'b u8) -> &'b u8 { | ^^ ^^ @@ -493,7 +493,7 @@ LL + fn multiple_inputs_output_not_elided<'b>(x: &u8, y: &'b u8, z: &'b u8) | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:467:22 + --> tests/ui/needless_lifetimes.rs:468:22 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ diff --git a/src/tools/miri/tests/pass/async-drop.rs b/src/tools/miri/tests/pass/async-drop.rs index 4fa84384d9bdd..6754d4ffbef10 100644 --- a/src/tools/miri/tests/pass/async-drop.rs +++ b/src/tools/miri/tests/pass/async-drop.rs @@ -1,6 +1,8 @@ -//@revisions: stack tree -//@compile-flags: -Zmiri-strict-provenance -//@[tree]compile-flags: -Zmiri-tree-borrows +//@ revisions: stack tree stackrelocate treerelocate +//@ compile-flags: -Zmiri-strict-provenance -Zpack-coroutine-layout=no +//@ [tree] compile-flags: -Zmiri-tree-borrows +//@ [stackrelocate] compile-flags: -Zpack-coroutine-layout=captures-only +//@ [treerelocate] compile-flags: -Zmiri-tree-borrows -Zpack-coroutine-layout=captures-only // WARNING: If you would ever want to modify this test, // please consider modifying rustc's async drop test at diff --git a/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/src/tools/miri/tests/pass/async-drop.treerelocate.stdout b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/tests/codegen-llvm/async-fn-debug.rs b/tests/codegen-llvm/async-fn-debug.rs index ed704c7cc8b92..a1fc4f14db0b1 100644 --- a/tests/codegen-llvm/async-fn-debug.rs +++ b/tests/codegen-llvm/async-fn-debug.rs @@ -38,17 +38,17 @@ async fn async_fn_test() { // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "3", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 13, +// CHECK-SAME: file: [[FILE]], line: 15, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 15, +// CHECK: [[S0:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend0", scope: [[GEN]], // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: [[S1:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend1", scope: [[GEN]], +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S0]] // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S1]] +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], +// CHECK-SAME: file: [[FILE]], line: 13, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: [[DISC]] = !DIDerivedType(tag: DW_TAG_member, name: "__state", scope: [[GEN]], diff --git a/tests/debuginfo/coroutine-objects.rs b/tests/debuginfo/coroutine-objects.rs index 7ead154cbdb1f..8b7a19b545dad 100644 --- a/tests/debuginfo/coroutine-objects.rs +++ b/tests/debuginfo/coroutine-objects.rs @@ -4,29 +4,29 @@ // with memory layout of discriminant for this particular enum // ensure that LLDB won't crash at least (like #57822). -//@ compile-flags:-g +//@ compile-flags: -g -Z pack-coroutine-layout=no //@ disable-gdb-pretty-printers // === GDB TESTS =================================================================================== // gdb-command:run // gdb-command:print b -// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{_ref__a: 0x[...]} +// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{a: 0x[...]} // gdb-command:continue // gdb-command:print b -// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, _ref__a: 0x[...]} +// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7} // gdb-command:continue // gdb-command:print b -// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, _ref__a: 0x[...]} +// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8} // gdb-command:continue // gdb-command:print b -// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned{_ref__a: 0x[...]} +// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned // === LLDB TESTS ================================================================================== // lldb-command:run // lldb-command:v b -// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { _ref__a = 0x[...] } $discr$ = [...] } +// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } // === CDB TESTS =================================================================================== diff --git a/tests/debuginfo/coroutine-objects_relocated.rs b/tests/debuginfo/coroutine-objects_relocated.rs new file mode 100644 index 0000000000000..83113f89607ed --- /dev/null +++ b/tests/debuginfo/coroutine-objects_relocated.rs @@ -0,0 +1,89 @@ +//@ min-lldb-version: 1800 + +// LLDB (18.1+) now supports DW_TAG_variant_part, but there is some bug in either compiler or LLDB +// with memory layout of discriminant for this particular enum +// ensure that LLDB won't crash at least (like #57822). + +//@ compile-flags: -g -Z pack-coroutine-layout=captures-only +//@ disable-gdb-pretty-printers + +// === GDB TESTS =================================================================================== + +// gdb-command:run +// gdb-command:print b +// gdb-check:$1 = coroutine_objects_relocated::main::{coroutine_env#0}::Unresumed{a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$2 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$3 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$4 = coroutine_objects_relocated::main::{coroutine_env#0}::Returned + +// === LLDB TESTS ================================================================================== + +// lldb-command:run +// lldb-command:v b +// lldb-check:(coroutine_objects_relocated::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } + +// === CDB TESTS =================================================================================== + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Unresumed [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend0 [Type: enum2$] +// cdb-check: [+0x[...]] c : 6 [Type: int] +// cdb-check: [+0x[...]] d : 7 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend1 [Type: enum2$] +// cdb-check: [+0x[...]] c : 7 [Type: int] +// cdb-check: [+0x[...]] d : 8 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Returned [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +use std::ops::Coroutine; +use std::pin::Pin; + +fn main() { + let mut a = 5; + let mut b = #[coroutine] + || { + let mut c = 6; + let mut d = 7; + + yield; + a += 1; + c += 1; + d += 1; + + yield; + println!("{} {} {}", a, c, d); + }; + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break +} + +#[inline(never)] +fn _zzz() { + () +} diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir index 050aac7c3ff05..32b3cd4e9c9c5 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _2; - debug x => ((*(_1.0: &mut {async fn body of a()})).0: T); + debug x => (((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -81,7 +81,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb11: { - drop(((*(_1.0: &mut {async fn body of a()})).0: T)) -> [return: bb10, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T)) -> [return: bb10, unwind unreachable]; } bb12: { diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir index 796e95ff3d825..fe1194837e9a2 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _2; - debug x => ((*(_1.0: &mut {async fn body of a()})).0: T); + debug x => (((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -95,7 +95,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb14: { - drop(((*(_1.0: &mut {async fn body of a()})).0: T)) -> [return: bb13, unwind: bb4]; + drop((((*(_1.0: &mut {async fn body of a()})) as variant#0).0: T)) -> [return: bb13, unwind: bb4]; } bb15 (cleanup): { diff --git a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir index 2e2876cb3fcf2..9908ea813f667 100644 --- a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir @@ -7,6 +7,7 @@ Panicked (2): [], }, storage_conflicts: BitMatrix(0x0) {}, + relocated_upvars: {}, } */ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { diff --git a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir index 20fc4112ef288..ec0314ea88fe8 100644 --- a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir @@ -40,13 +40,14 @@ Unresumed(0): [], Returned (1): [], Panicked (2): [], - Suspend0 (3): [_s0], - Suspend1 (4): [_s1], + Suspend0 (3): [_s1], + Suspend1 (4): [_s0], }, storage_conflicts: BitMatrix(2x2) { (_s0, _s0), (_s1, _s1), }, + relocated_upvars: {}, } */ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> Poll<()> { @@ -87,14 +88,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> let mut _37: (); let mut _38: u32; scope 1 { - debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); + debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); let _17: (); scope 2 { debug result => _17; } } scope 3 { - debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); + debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); let _33: (); scope 4 { debug result => _33; @@ -121,7 +122,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_5); PlaceMention(_4); nop; - (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}) = move _4; + (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}) = move _4; goto -> bb4; } @@ -131,7 +132,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_10); StorageLive(_11); StorageLive(_12); - _12 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); + _12 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); _11 = &mut (*_12); _10 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _11) -> [return: bb5, unwind unreachable]; } @@ -178,7 +179,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_4); StorageDead(_19); StorageDead(_20); - discriminant((*(_1.0: &mut {async fn body of b()}))) = 3; + discriminant((*(_1.0: &mut {async fn body of b()}))) = 4; return; } @@ -191,7 +192,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_12); StorageDead(_9); StorageDead(_8); - drop((((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; } bb11: { @@ -223,7 +224,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_22); PlaceMention(_21); nop; - (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}) = move _21; + (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}) = move _21; goto -> bb16; } @@ -233,7 +234,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_26); StorageLive(_27); StorageLive(_28); - _28 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); + _28 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); _27 = &mut (*_28); _26 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _27) -> [return: bb17, unwind unreachable]; } @@ -275,7 +276,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_21); StorageDead(_35); StorageDead(_36); - discriminant((*(_1.0: &mut {async fn body of b()}))) = 4; + discriminant((*(_1.0: &mut {async fn body of b()}))) = 3; return; } @@ -288,7 +289,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_28); StorageDead(_25); StorageDead(_24); - drop((((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; } bb22: { @@ -320,6 +321,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> } bb27: { + StorageLive(_21); + StorageLive(_35); + StorageLive(_36); + _35 = move _2; + goto -> bb22; + } + + bb28: { StorageLive(_3); StorageLive(_4); StorageLive(_19); @@ -328,14 +337,6 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> goto -> bb11; } - bb28: { - StorageLive(_21); - StorageLive(_35); - StorageLive(_36); - _35 = move _2; - goto -> bb22; - } - bb29: { assert(const false, "`async fn` resumed after completion") -> [success: bb29, unwind unreachable]; } diff --git a/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir index d8fdb446135b3..db0fe510c4f86 100644 --- a/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir +++ b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir @@ -20,6 +20,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:18:5: 18:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> { diff --git a/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir index dd17afad656b8..bd3d2c243d276 100644 --- a/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir +++ b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir @@ -20,6 +20,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#1}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:25:5: 25:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> { diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir new file mode 100644 index 0000000000000..d6067ec2ba63f --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir @@ -0,0 +1,115 @@ +// MIR for `main::{closure#0}` after RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*_9); + debug x => _9; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; + let mut _9: &mut std::string::String; + let _10: &mut std::string::String; + + bb0: { + StorageLive(_9); + StorageLive(_10); + _10 = move (_1.0: &mut std::string::String); + _9 = move _10; + StorageDead(_10); + goto -> bb18; + } + + bb1: { + _6 = deref_copy _9; + drop((*_6)) -> [return: bb2, unwind unreachable]; + } + + bb2: { + _7 = deref_copy _9; + (*_7) = move _3; + goto -> bb3; + } + + bb3: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb4, drop: bb6]; + } + + bb4: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + goto -> bb13; + } + + bb5: { + return; + } + + bb6: { + StorageDead(_5); + StorageDead(_4); + goto -> bb7; + } + + bb7: { + goto -> bb17; + } + + bb8: { + coroutine_drop; + } + + bb9 (cleanup): { + unreachable; + } + + bb10 (cleanup): { + StorageDead(_9); + goto -> bb9; + } + + bb11 (cleanup): { + goto -> bb10; + } + + bb12: { + StorageDead(_9); + goto -> bb5; + } + + bb13: { + goto -> bb12; + } + + bb14 (cleanup): { + StorageDead(_9); + goto -> bb9; + } + + bb15 (cleanup): { + goto -> bb14; + } + + bb16: { + StorageDead(_9); + goto -> bb8; + } + + bb17: { + goto -> bb16; + } + + bb18: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind unreachable]; + } +} diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir new file mode 100644 index 0000000000000..7c90429ef8dc6 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir @@ -0,0 +1,139 @@ +// MIR for `main::{closure#0}` after RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*_9); + debug x => _9; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; + let mut _9: &mut std::string::String; + let _10: &mut std::string::String; + + bb0: { + StorageLive(_9); + StorageLive(_10); + _10 = move (_1.0: &mut std::string::String); + _9 = move _10; + StorageDead(_10); + goto -> bb23; + } + + bb1: { + _6 = deref_copy _9; + drop((*_6)) -> [return: bb2, unwind: bb3]; + } + + bb2: { + _7 = deref_copy _9; + (*_7) = move _3; + goto -> bb4; + } + + bb3 (cleanup): { + _8 = deref_copy _9; + (*_8) = move _3; + goto -> bb10; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + goto -> bb15; + } + + bb6: { + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb8; + } + + bb8: { + goto -> bb19; + } + + bb9: { + coroutine_drop; + } + + bb10 (cleanup): { + StorageDead(_3); + goto -> bb22; + } + + bb11 (cleanup): { + resume; + } + + bb12 (cleanup): { + StorageDead(_9); + goto -> bb11; + } + + bb13 (cleanup): { + goto -> bb12; + } + + bb14: { + StorageDead(_9); + goto -> bb6; + } + + bb15: { + goto -> bb14; + } + + bb16 (cleanup): { + StorageDead(_9); + goto -> bb11; + } + + bb17 (cleanup): { + goto -> bb16; + } + + bb18: { + StorageDead(_9); + goto -> bb9; + } + + bb19: { + goto -> bb18; + } + + bb20 (cleanup): { + terminate(cleanup); + } + + bb21 (cleanup): { + StorageDead(_9); + goto -> bb11; + } + + bb22 (cleanup): { + goto -> bb21; + } + + bb23: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb10]; + } +} diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir new file mode 100644 index 0000000000000..2fd8713094c05 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir @@ -0,0 +1,63 @@ +// MIR for `main::{closure#0}` before RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*(_1.0: &mut std::string::String)); + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; + + bb0: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind unreachable]; + } + + bb1: { + _6 = deref_copy (_1.0: &mut std::string::String); + drop((*_6)) -> [return: bb2, unwind unreachable]; + } + + bb2: { + _7 = deref_copy (_1.0: &mut std::string::String); + (*_7) = move _3; + goto -> bb3; + } + + bb3: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb4, drop: bb6]; + } + + bb4: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + drop(_1) -> [return: bb5, unwind unreachable]; + } + + bb5: { + return; + } + + bb6: { + StorageDead(_5); + StorageDead(_4); + goto -> bb7; + } + + bb7: { + drop(_1) -> [return: bb8, unwind unreachable]; + } + + bb8: { + coroutine_drop; + } +} diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir new file mode 100644 index 0000000000000..eb4762a233a9f --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir @@ -0,0 +1,78 @@ +// MIR for `main::{closure#0}` before RelocateUpvars + +fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:12:5: 12:7}, _2: ()) -> () +yields () + { + debug x => (*(_1.0: &mut std::string::String)); + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; + + bb0: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb10]; + } + + bb1: { + _6 = deref_copy (_1.0: &mut std::string::String); + drop((*_6)) -> [return: bb2, unwind: bb3]; + } + + bb2: { + _7 = deref_copy (_1.0: &mut std::string::String); + (*_7) = move _3; + goto -> bb4; + } + + bb3 (cleanup): { + _8 = deref_copy (_1.0: &mut std::string::String); + (*_8) = move _3; + goto -> bb10; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + drop(_1) -> [return: bb6, unwind: bb11]; + } + + bb6: { + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb8; + } + + bb8: { + drop(_1) -> [return: bb9, unwind: bb11]; + } + + bb9: { + coroutine_drop; + } + + bb10 (cleanup): { + StorageDead(_3); + drop(_1) -> [return: bb11, unwind terminate(cleanup)]; + } + + bb11 (cleanup): { + resume; + } +} diff --git a/tests/mir-opt/coroutine_relocate_upvars.rs b/tests/mir-opt/coroutine_relocate_upvars.rs new file mode 100644 index 0000000000000..42c4ca896a3e8 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.rs @@ -0,0 +1,16 @@ +// skip-filecheck +//@ compile-flags: -Zpack-coroutine-layout=captures-only +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY + +// EMIT_MIR coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.mir +// EMIT_MIR coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.mir +fn main() { + let mut x = String::new(); + let gen_ = #[coroutine] + || { + x = String::new(); + yield; + }; +} diff --git a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir index 9905cc3e00f99..7620919f2915e 100644 --- a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir @@ -19,6 +19,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}>, _2: u8) -> CoroutineState<(), ()> { diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index 0203ff52d63f8..584a0bbb37afe 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -177,7 +177,7 @@ + bb3: { + _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_33) as variant#3).0: ActionPermit<'_, T>) = move ((*_34).0: ActionPermit<'_, T>); ++ (((*_33) as variant#3).0: ActionPermit<'_, T>) = move (((*_34) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index 6c8cad3e992e7..ebd23164f44a6 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -194,7 +194,7 @@ + bb5: { + _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_33) as variant#3).0: ActionPermit<'_, T>) = move ((*_34).0: ActionPermit<'_, T>); ++ (((*_33) as variant#3).0: ActionPermit<'_, T>) = move (((*_34) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout similarity index 94% rename from tests/ui/async-await/async-drop/async-drop-initial.run.stdout rename to tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout index 9cae4331caf92..792880d835935 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout +++ b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout @@ -20,3 +20,4 @@ AsyncInt::Dropper::poll: 18 AsyncInt::Dropper::poll: 19 AsyncInt::Dropper::poll: 20 AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout new file mode 100644 index 0000000000000..792880d835935 --- /dev/null +++ b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout @@ -0,0 +1,23 @@ +AsyncInt::Dropper::poll: 0 +AsyncInt::Dropper::poll: 1 +AsyncInt::Dropper::poll: 2 +AsyncInt::Dropper::poll: 3 +AsyncInt::Dropper::poll: 4 +AsyncStruct::Dropper::poll: 6 +AsyncInt::Dropper::poll: 7 +AsyncInt::Dropper::poll: 8 +AsyncReference::Dropper::poll: 10 +AsyncInt::Dropper::poll: 11 +AsyncEnum(A)::Dropper::poll: 12 +SyncInt::drop: 12 +AsyncEnum(B)::Dropper::poll: 13 +AsyncInt::Dropper::poll: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::Dropper::poll: 16 +SyncInt::drop: 17 +AsyncInt::Dropper::poll: 18 +AsyncInt::Dropper::poll: 19 +AsyncInt::Dropper::poll: 20 +AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.rs b/tests/ui/async-await/async-drop/async-drop-initial.rs index cd33c143fba01..f44dc88000390 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.rs +++ b/tests/ui/async-await/async-drop/async-drop-initial.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ run-pass //@ check-run-results @@ -11,26 +14,32 @@ //@ edition: 2021 // FIXME(zetanumbers): consider AsyncDestruct::async_drop cleanup tests -use core::future::{async_drop_in_place, AsyncDrop, Future}; +use core::future::{AsyncDrop, Future, async_drop_in_place}; use core::hint::black_box; use core::mem::{self, ManuallyDrop}; -use core::pin::{pin, Pin}; +use core::pin::{Pin, pin}; +use core::sync::atomic::{AtomicBool, Ordering}; use core::task::{Context, Poll, Waker}; -async fn test_async_drop(x: T, _size: usize) { +static PASS: AtomicBool = AtomicBool::new(false); + +async fn test_async_drop(x: T, #[allow(unused)] expect: usize) { let mut x = mem::MaybeUninit::new(x); let dtor = pin!(unsafe { async_drop_in_place(x.as_mut_ptr()) }); // FIXME(zetanumbers): This check fully depends on the layout of // the coroutine state, since async destructor combinators are just // async functions. + #[allow(unused)] + let got = mem::size_of_val(&*dtor); #[cfg(target_pointer_width = "64")] - assert_eq!( - mem::size_of_val(&*dtor), - _size, - "sizes did not match for async destructor of type {}", - core::any::type_name::(), - ); + if expect != got { + println!( + "sizes did not match for async destructor of type {}, expect {expect}, got {got}", + core::any::type_name::() + ); + PASS.store(false, Ordering::Relaxed); + } test_idempotency(dtor).await; } @@ -47,27 +56,29 @@ where } fn main() { + PASS.store(true, Ordering::Relaxed); let waker = Waker::noop(); let mut cx = Context::from_waker(&waker); let i = 13; let fut = pin!(async { - test_async_drop(Int(0), 16).await; - test_async_drop(AsyncInt(0), 32).await; - test_async_drop([AsyncInt(1), AsyncInt(2)], 104).await; - test_async_drop((AsyncInt(3), AsyncInt(4)), 120).await; - test_async_drop(5, 16).await; + test_async_drop(Int(0), [16, 24][cfg!(classic) as usize]).await; + test_async_drop(AsyncInt(0), [32, 48][cfg!(classic) as usize]).await; + test_async_drop([AsyncInt(1), AsyncInt(2)], [96, 136][cfg!(classic) as usize]).await; + test_async_drop((AsyncInt(3), AsyncInt(4)), [112, 168][cfg!(classic) as usize]).await; + test_async_drop(5, [16, 24][cfg!(classic) as usize]).await; let j = 42; - test_async_drop(&i, 16).await; - test_async_drop(&j, 16).await; + test_async_drop(&i, [16, 24][cfg!(classic) as usize]).await; + test_async_drop(&j, [16, 24][cfg!(classic) as usize]).await; test_async_drop( AsyncStruct { b: AsyncInt(8), a: AsyncInt(7), i: 6 }, - 136, - ).await; - test_async_drop(ManuallyDrop::new(AsyncInt(9)), 16).await; + if cfg!(panic = "unwind") { [128, 192][cfg!(classic) as usize] } else { 136 }, + ) + .await; + test_async_drop(ManuallyDrop::new(AsyncInt(9)), [16, 24][cfg!(classic) as usize]).await; let foo = AsyncInt(10); - test_async_drop(AsyncReference { foo: &foo }, 32).await; + test_async_drop(AsyncReference { foo: &foo }, [32, 48][cfg!(classic) as usize]).await; let _ = ManuallyDrop::new(foo); let foo = AsyncInt(11); @@ -77,17 +88,17 @@ fn main() { let foo = AsyncInt(10); foo }, - 48, + [48, 72][cfg!(classic) as usize], ) .await; - test_async_drop(AsyncEnum::A(AsyncInt(12)), 104).await; - test_async_drop(AsyncEnum::B(SyncInt(13)), 104).await; + test_async_drop(AsyncEnum::A(AsyncInt(12)), [96, 144][cfg!(classic) as usize]).await; + test_async_drop(AsyncEnum::B(SyncInt(13)), [96, 144][cfg!(classic) as usize]).await; - test_async_drop(SyncInt(14), 16).await; + test_async_drop(SyncInt(14), [16, 24][cfg!(classic) as usize]).await; test_async_drop( SyncThenAsync { i: 15, a: AsyncInt(16), b: SyncInt(17), c: AsyncInt(18) }, - 120, + [112, 168][cfg!(classic) as usize], ) .await; @@ -104,14 +115,16 @@ fn main() { black_box(core::future::ready(())).await; foo }, - 48, + [48, 72][cfg!(classic) as usize], ) .await; - test_async_drop(AsyncUnion { signed: 21 }, 32).await; + test_async_drop(AsyncUnion { signed: 21 }, [32, 48][cfg!(classic) as usize]).await; + test_async_drop(AsyncUnion { signed: 21 }, [32, 48][cfg!(classic) as usize]).await; }); let res = fut.poll(&mut cx); assert_eq!(res, Poll::Ready(())); + assert!(PASS.load(Ordering::Relaxed)); } struct AsyncInt(i32); @@ -228,19 +241,13 @@ union AsyncUnion { impl Drop for AsyncUnion { fn drop(&mut self) { - println!( - "AsyncUnion::drop: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::drop: {}, {}", unsafe { self.signed }, unsafe { self.unsigned },); } } impl AsyncDrop for AsyncUnion { async fn drop(self: Pin<&mut Self>) { - println!( - "AsyncUnion::Dropper::poll: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::Dropper::poll: {}, {}", unsafe { self.signed }, unsafe { + self.unsigned + },); } } diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout new file mode 100644 index 0000000000000..12e7fa9f95155 --- /dev/null +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout @@ -0,0 +1,113 @@ +print-type-size type: `{async fn body of test()}`: 6150 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 6149 bytes +print-type-size local `.__awaitee`: 6149 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 6149 bytes, alignment: 1 bytes +print-type-size field `.value`: 6149 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 6149 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 6149 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 6149 bytes +print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 6149 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 2049 bytes +print-type-size local `.fut`: 2049 bytes +print-type-size upvar `.fut`: 2049 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 4100 bytes +print-type-size padding: 2049 bytes +print-type-size local `.fut`: 2049 bytes, alignment: 1 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Suspend1`: 6148 bytes +print-type-size padding: 4098 bytes +print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool +print-type-size local `.__awaitee`: 2049 bytes, type: {async fn body of big_fut()} +print-type-size variant `Suspend2`: 4100 bytes +print-type-size padding: 2049 bytes +print-type-size local `.fut`: 2049 bytes, alignment: 1 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Returned`: 2049 bytes +print-type-size variant `Panicked`: 2049 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 2049 bytes, alignment: 1 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 2049 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2049 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `{async fn body of big_fut()}`: 2049 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size upvar `.arg`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size end padding: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of big_fut()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of wait()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::mem::ManuallyDrop`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout similarity index 77% rename from tests/ui/async-await/future-sizes/async-awaiting-fut.stdout rename to tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout index b30c15bcbe6ed..2c658c5f925a0 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout @@ -1,39 +1,36 @@ -print-type-size type: `{async fn body of test()}`: 3078 bytes, alignment: 1 bytes +print-type-size type: `{async fn body of test()}`: 2053 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 0 bytes -print-type-size variant `Suspend0`: 3077 bytes -print-type-size local `.__awaitee`: 3077 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} +print-type-size variant `Suspend0`: 2052 bytes +print-type-size local `.__awaitee`: 2052 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} print-type-size variant `Returned`: 0 bytes print-type-size variant `Panicked`: 0 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 3077 bytes, alignment: 1 bytes -print-type-size field `.value`: 3077 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 3077 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 3077 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2052 bytes print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 3077 bytes -print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 3077 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 2052 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Suspend0`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Unresumed`: 2051 bytes +print-type-size upvar `.fut`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size padding: 1026 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 1027 bytes print-type-size local `.fut`: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} -print-type-size variant `Suspend1`: 3076 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Suspend1`: 2051 bytes print-type-size padding: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of big_fut()} -print-type-size variant `Suspend2`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Suspend2`: 1027 bytes print-type-size local `.fut`: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} -print-type-size variant `Returned`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Panicked`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes @@ -43,11 +40,16 @@ print-type-size field `.value`: 1025 bytes print-type-size type: `{async fn body of big_fut()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes print-type-size field `.waker`: 8 bytes print-type-size field `.local_waker`: 8 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs index 7113f591630d1..d646c7b8f039a 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type lib diff --git a/tests/ui/async-await/future-sizes/future-as-arg.rs b/tests/ui/async-await/future-sizes/future-as-arg.rs index 317c17b9b3e59..3bbbadc2099df 100644 --- a/tests/ui/async-await/future-sizes/future-as-arg.rs +++ b/tests/ui/async-await/future-sizes/future-as-arg.rs @@ -8,9 +8,10 @@ async fn use_future(fut: impl std::future::Future) { } fn main() { - let actual = std::mem::size_of_val( - &use_future(use_future(use_future(use_future(use_future(test([0; 16]))))))); + let actual = std::mem::size_of_val(&use_future(use_future(use_future(use_future( + use_future(test([0; 16])), + ))))); // Not using an exact number in case it slightly changes over different commits - let expected = 550; - assert!(actual > expected, "expected: >{expected}, actual: {actual}"); + let expected = 21; + assert!(actual >= expected, "expected: >{expected}, actual: {actual}"); } diff --git a/tests/ui/async-await/future-sizes/large-arg.classic.stdout b/tests/ui/async-await/future-sizes/large-arg.classic.stdout new file mode 100644 index 0000000000000..4529d4a2e5814 --- /dev/null +++ b/tests/ui/async-await/future-sizes/large-arg.classic.stdout @@ -0,0 +1,106 @@ +print-type-size type: `{async fn body of test()}`: 4100 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 4099 bytes +print-type-size local `.__awaitee`: 4099 bytes, type: {async fn body of a<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 4099 bytes, alignment: 1 bytes +print-type-size field `.value`: 4099 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 4099 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 4099 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 4099 bytes +print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 4099 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 4098 bytes +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 3074 bytes, alignment: 1 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 3074 bytes, alignment: 1 bytes +print-type-size field `.value`: 3074 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 3074 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 3074 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 3074 bytes +print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 3074 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 3073 bytes +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 2049 bytes, alignment: 1 bytes, type: {async fn body of c<[u8; 1024]>()} +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 2049 bytes, alignment: 1 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 2049 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2049 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 2049 bytes +print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 2049 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Returned`: 1024 bytes +print-type-size variant `Panicked`: 1024 bytes +print-type-size end padding: 1024 bytes +print-type-size type: `std::task::Poll<[u8; 1024]>`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 1024 bytes +print-type-size field `.0`: 1024 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of a<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of b<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of c<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.stdout b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout similarity index 68% rename from tests/ui/async-await/future-sizes/large-arg.stdout rename to tests/ui/async-await/future-sizes/large-arg.relocate.stdout index e00420d1493f4..53465757a3d1e 100644 --- a/tests/ui/async-await/future-sizes/large-arg.stdout +++ b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout @@ -1,44 +1,40 @@ -print-type-size type: `{async fn body of test()}`: 3076 bytes, alignment: 1 bytes +print-type-size type: `{async fn body of test()}`: 1028 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 0 bytes -print-type-size variant `Suspend0`: 3075 bytes -print-type-size local `.__awaitee`: 3075 bytes, type: {async fn body of a<[u8; 1024]>()} +print-type-size variant `Suspend0`: 1027 bytes +print-type-size local `.__awaitee`: 1027 bytes, type: {async fn body of a<[u8; 1024]>()} print-type-size variant `Returned`: 0 bytes print-type-size variant `Panicked`: 0 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 3075 bytes, alignment: 1 bytes -print-type-size field `.value`: 3075 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 3075 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 3075 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1027 bytes print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 3075 bytes -print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 3075 bytes, alignment: 1 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 1027 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Suspend0`: 3074 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 2050 bytes, type: {async fn body of b<[u8; 1024]>()} -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes -print-type-size field `.value`: 2050 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 2050 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1026 bytes +print-type-size local `.__awaitee`: 1026 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1026 bytes print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 2050 bytes -print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 2050 bytes, alignment: 1 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 1026 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Suspend0`: 2049 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1025 bytes print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes @@ -53,11 +49,16 @@ print-type-size variant `Pending`: 0 bytes print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes print-type-size field `.waker`: 8 bytes print-type-size field `.local_waker`: 8 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.rs b/tests/ui/async-await/future-sizes/large-arg.rs index b05a2b7191512..2dd440b3f3a7f 100644 --- a/tests/ui/async-await/future-sizes/large-arg.rs +++ b/tests/ui/async-await/future-sizes/large-arg.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type=lib diff --git a/tests/ui/async-await/issue-70818.stderr b/tests/ui/async-await/issue-70818.stderr index 07fd20cdd77ea..efc412cef098b 100644 --- a/tests/ui/async-await/issue-70818.stderr +++ b/tests/ui/async-await/issue-70818.stderr @@ -5,10 +5,10 @@ LL | async { (ty, ty1) } | ^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` note: required by a bound in an opaque type --> $DIR/issue-70818.rs:4:69 | @@ -26,10 +26,10 @@ LL | fn foo(ty: T, ty1: U) -> impl Future + Send { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` help: consider restricting type parameter `U` with trait `Send` | LL | fn foo(ty: T, ty1: U) -> impl Future + Send { diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs index f6c9fdd6d6806..63adb5a725172 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs @@ -9,7 +9,7 @@ pub async fn async_fn(x: &mut i32) -> &i32 { y } -pub fn async_closure(x: &mut i32) -> impl Future { +pub fn async_closure(x: &mut i32) -> impl Future { (async move || { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -19,7 +19,7 @@ pub fn async_closure(x: &mut i32) -> impl Future { })() } -pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { (async move || -> &i32 { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -29,7 +29,7 @@ pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future impl Future { +pub fn async_block(x: &mut i32) -> impl Future { async move { let y = &*x; *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr index e1f268116fc54..cb559898c2fc5 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr @@ -44,7 +44,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:13:5 | -LL | pub fn async_closure(x: &mut i32) -> impl Future { +LL | pub fn async_closure(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || { LL | || @@ -93,7 +93,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:23:5 | -LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || -> &i32 { LL | || diff --git a/tests/ui/async-await/issue-86507.rs b/tests/ui/async-await/issue-86507.rs index 484122a1ddcfd..6dbebc9328677 100644 --- a/tests/ui/async-await/issue-86507.rs +++ b/tests/ui/async-await/issue-86507.rs @@ -12,9 +12,14 @@ trait Foo { impl Foo for () { fn bar<'me, 'async_trait, T: Send>(x: &'me T) + //~^ NOTE captured value is not `Send` + //~| NOTE has type `&T` which is not `Send` -> Pin + Send + 'async_trait>> where 'me:'async_trait { - Box::pin( //~ ERROR future cannot be sent between threads safely + Box::pin( + //~^ ERROR future cannot be sent between threads safely + //~| NOTE future created by async block is not `Send` + //~| NOTE required for the cast from async move { let x = x; } diff --git a/tests/ui/async-await/issue-86507.stderr b/tests/ui/async-await/issue-86507.stderr index c71801dcfc854..32b9f795ee79d 100644 --- a/tests/ui/async-await/issue-86507.stderr +++ b/tests/ui/async-await/issue-86507.stderr @@ -1,19 +1,17 @@ error: future cannot be sent between threads safely - --> $DIR/issue-86507.rs:17:13 + --> $DIR/issue-86507.rs:19:13 | LL | / Box::pin( -LL | | async move { -LL | | let x = x; -LL | | } +... | LL | | ) | |_____________^ future created by async block is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/issue-86507.rs:19:29 + --> $DIR/issue-86507.rs:14:40 | -LL | let x = x; - | ^ has type `&T` which is not `Send`, because `T` is not `Sync` - = note: required for the cast from `Pin>` to `Pin + Send>>` +LL | fn bar<'me, 'async_trait, T: Send>(x: &'me T) + | ^ has type `&T` which is not `Send`, because `T` is not `Sync` + = note: required for the cast from `Pin>` to `Pin + Send>>` help: consider further restricting type parameter `T` with trait `Sync` | LL | fn bar<'me, 'async_trait, T: Send + std::marker::Sync>(x: &'me T) diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs index 9317ab5cd2756..d5e849b77e812 100644 --- a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs +++ b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs @@ -2,12 +2,25 @@ //@ edition: 2018 //@ compile-flags: --target thumbv8m.main-none-eabi --crate-type lib //@ needs-llvm-components: arm -#![feature(cmse_nonsecure_entry, c_variadic, no_core, lang_items)] +#![feature(cmse_nonsecure_entry, rustc_attrs, c_variadic, no_core, lang_items, transparent_unions)] #![no_core] extern crate minicore; use minicore::*; +#[lang = "clone"] +pub trait Clone: Sized {} + +#[lang = "maybe_uninit"] +#[repr(transparent)] +#[rustc_pub_transparent] +pub union MaybeUninit { + uninit: (), + value: ManuallyDrop, +} +impl Copy for MaybeUninit {} +impl Clone for MaybeUninit {} + #[lang = "va_list"] struct VaList(*mut u8); diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr index 948f8f5747b0e..060dc4129de2d 100644 --- a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr +++ b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr @@ -1,5 +1,5 @@ error: `...` is not supported for `extern "cmse-nonsecure-entry"` functions - --> $DIR/c-variadic.rs:14:60 + --> $DIR/c-variadic.rs:27:60 | LL | unsafe extern "cmse-nonsecure-entry" fn c_variadic(_: u32, _: ...) { | ----------------------------- ^^^^^^ @@ -9,13 +9,13 @@ LL | unsafe extern "cmse-nonsecure-entry" fn c_variadic(_: u32, _: ...) { = help: only `extern "C"` and `extern "C-unwind"` functions may have a C variable argument list error: functions cannot be both `async` and C-variadic - --> $DIR/c-variadic.rs:19:1 + --> $DIR/c-variadic.rs:32:1 | LL | async unsafe extern "cmse-nonsecure-entry" fn async_and_c_variadic(_: ...) { | ^^^^^ `async` because of this ^^^^^^ C-variadic because of this error: `...` is not supported for `extern "cmse-nonsecure-entry"` functions - --> $DIR/c-variadic.rs:19:68 + --> $DIR/c-variadic.rs:32:68 | LL | async unsafe extern "cmse-nonsecure-entry" fn async_and_c_variadic(_: ...) { | ----------------------------- ^^^^^^ diff --git a/tests/ui/coroutine/clone-impl.stderr b/tests/ui/coroutine/clone-impl.classic.stderr similarity index 58% rename from tests/ui/coroutine/clone-impl.stderr rename to tests/ui/coroutine/clone-impl.classic.stderr index f316902a42d08..73513cf8df612 100644 --- a/tests/ui/coroutine/clone-impl.stderr +++ b/tests/ui/coroutine/clone-impl.classic.stderr @@ -1,81 +1,81 @@ -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` - --> $DIR/clone-impl.rs:48:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` ... LL | check_copy(&gen_clone_0); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:46:14 + --> $DIR/clone-impl.rs:45:9 | -LL | drop(clonable_0); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_0: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` - --> $DIR/clone-impl.rs:60:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` | note: coroutine does not implement `Copy` as this value is used across a yield - --> $DIR/clone-impl.rs:57:9 + --> $DIR/clone-impl.rs:60:9 | LL | let v = vec!['a']; | - has type `Vec` which does not implement `Copy` LL | yield; | ^^^^^ yield occurs here, with `v` maybe used later note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` - --> $DIR/clone-impl.rs:74:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:72:14 + --> $DIR/clone-impl.rs:69:9 | -LL | drop(clonable_1); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_1: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:86:5 +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_copy(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Copy` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` @@ -85,22 +85,22 @@ LL + #[derive(Copy)] LL | struct NonClone; | -error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:88:5 +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_clone(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Clone` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` | note: captured value does not implement `Clone` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` note: required by a bound in `check_clone` - --> $DIR/clone-impl.rs:93:19 + --> $DIR/clone-impl.rs:96:19 | LL | fn check_clone(_x: &T) {} | ^^^^^ required by this bound in `check_clone` diff --git a/tests/ui/coroutine/clone-impl.relocate.stderr b/tests/ui/coroutine/clone-impl.relocate.stderr new file mode 100644 index 0000000000000..73513cf8df612 --- /dev/null +++ b/tests/ui/coroutine/clone-impl.relocate.stderr @@ -0,0 +1,115 @@ +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` +... +LL | check_copy(&gen_clone_0); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:45:9 + | +LL | let clonable_0: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` + | +note: coroutine does not implement `Copy` as this value is used across a yield + --> $DIR/clone-impl.rs:60:9 + | +LL | let v = vec!['a']; + | - has type `Vec` which does not implement `Copy` +LL | yield; + | ^^^^^ yield occurs here, with `v` maybe used later +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:69:9 + | +LL | let clonable_1: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_copy(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:83:9 + | +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` +help: consider annotating `NonClone` with `#[derive(Copy)]` + | +LL + #[derive(Copy)] +LL | struct NonClone; + | + +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_clone(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` + | +note: captured value does not implement `Clone` + --> $DIR/clone-impl.rs:83:9 + | +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +note: required by a bound in `check_clone` + --> $DIR/clone-impl.rs:96:19 + | +LL | fn check_clone(_x: &T) {} + | ^^^^^ required by this bound in `check_clone` +help: consider annotating `NonClone` with `#[derive(Clone)]` + | +LL + #[derive(Clone)] +LL | struct NonClone; + | + +error: aborting due to 5 previous errors + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/coroutine/clone-impl.rs b/tests/ui/coroutine/clone-impl.rs index 9e04e256fc11e..17723b56449b3 100644 --- a/tests/ui/coroutine/clone-impl.rs +++ b/tests/ui/coroutine/clone-impl.rs @@ -1,7 +1,10 @@ // gate-test-coroutine_clone // Verifies that non-static coroutines can be cloned/copied if all their upvars and locals held // across awaits can be cloned/copied. -//@compile-flags: --diagnostic-width=300 +//@ revisions: classic relocate +//@ compile-flags: --diagnostic-width=300 +//@ [classic] compile-flags: -Zpack-coroutine-layout=no +//@ [relocate] compile-flags: -Zpack-coroutine-layout=captures-only #![feature(coroutines, coroutine_clone, stmt_expr_attributes)] diff --git a/tests/ui/coroutine/delayed-obligations-emit.next.stderr b/tests/ui/coroutine/delayed-obligations-emit.next.stderr index 3a3663398c9a7..af7dfc75963c2 100644 --- a/tests/ui/coroutine/delayed-obligations-emit.next.stderr +++ b/tests/ui/coroutine/delayed-obligations-emit.next.stderr @@ -4,6 +4,7 @@ error[E0275]: overflow evaluating the requirement `{async block@$DIR/delayed-obl LL | spawn(async { build_dependencies().await }); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | + = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`delayed_obligations_emit`) note: required by a bound in `spawn` --> $DIR/delayed-obligations-emit.rs:31:13 | diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs index 0f9c56786da06..c862ba658aa01 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs @@ -1,16 +1,13 @@ #![feature(coroutines, stmt_expr_attributes)] fn foo(x: &i32) { - // In this case, a reference to `b` escapes the coroutine, but not - // because of a yield. We see that there is no yield in the scope of - // `b` and give the more generic error message. let mut a = &3; - let mut b = #[coroutine] + let b = #[coroutine] move || { yield (); let b = 5; a = &b; - //~^ ERROR borrowed data escapes outside of coroutine + //~^ ERROR: borrowed data escapes outside of coroutine }; } diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr index 6fa7082c0b8b7..41b118d9f8f3a 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr @@ -1,5 +1,5 @@ error[E0521]: borrowed data escapes outside of coroutine - --> $DIR/ref-escapes-but-not-over-yield.rs:12:9 + --> $DIR/ref-escapes-but-not-over-yield.rs:9:9 | LL | let mut a = &3; | ----- `a` declared here, outside of the coroutine body diff --git a/tests/ui/coroutine/ref-upvar-not-send.rs b/tests/ui/coroutine/ref-upvar-not-send.rs index 89bb5e5495f45..a843e9ba29a64 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.rs +++ b/tests/ui/coroutine/ref-upvar-not-send.rs @@ -11,21 +11,27 @@ fn assert_send(_: T) {} fn main() { let x: &*mut () = &std::ptr::null_mut(); + //~^ NOTE has type `&*mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` let y: &mut *mut () = &mut std::ptr::null_mut(); - assert_send(#[coroutine] move || { + //~^ NOTE has type `&mut *mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _x = x; - }); - //~^^ NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - //~| NOTE has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` - assert_send(#[coroutine] move || { + #[coroutine] + move || { + yield; + let _x = x; + }, + ); + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _y = y; - }); - //~^^ NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - //~| NOTE has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` + #[coroutine] + move || { + yield; + let _y = y; + }, + ); } diff --git a/tests/ui/coroutine/ref-upvar-not-send.stderr b/tests/ui/coroutine/ref-upvar-not-send.stderr index 3a5e8ec4dab0c..1197583c0cc08 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.stderr +++ b/tests/ui/coroutine/ref-upvar-not-send.stderr @@ -1,20 +1,21 @@ error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:15:5 + --> $DIR/ref-upvar-not-send.rs:19:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _x = x; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | = help: the trait `Sync` is not implemented for `*mut ()` note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/ref-upvar-not-send.rs:19:18 + --> $DIR/ref-upvar-not-send.rs:13:9 | -LL | let _x = x; - | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` +LL | let x: &*mut () = &std::ptr::null_mut(); + | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | @@ -22,22 +23,23 @@ LL | fn assert_send(_: T) {} | ^^^^ required by this bound in `assert_send` error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:23:5 + --> $DIR/ref-upvar-not-send.rs:28:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _y = y; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | - = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:23:30: 23:37}`, the trait `Send` is not implemented for `*mut ()` + = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:32:9: 32:16}`, the trait `Send` is not implemented for `*mut ()` note: captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - --> $DIR/ref-upvar-not-send.rs:27:18 + --> $DIR/ref-upvar-not-send.rs:16:9 | -LL | let _y = y; - | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` +LL | let y: &mut *mut () = &mut std::ptr::null_mut(); + | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | diff --git a/tests/ui/coroutine/unsized-capture-across-yield.rs b/tests/ui/coroutine/unsized-capture-across-yield.rs index ee27ea064ec23..60c8f69bdb8f0 100644 --- a/tests/ui/coroutine/unsized-capture-across-yield.rs +++ b/tests/ui/coroutine/unsized-capture-across-yield.rs @@ -4,7 +4,8 @@ use std::ops::Coroutine; fn capture() -> impl Coroutine { - let b: [u8] = *(Box::new([]) as Box<[u8]>); //~ERROR he size for values of type `[u8]` cannot be known at compilation time + let b: [u8] = *(Box::new([]) as Box<[u8]>); + //~^ ERROR the size for values of type `[u8]` cannot be known at compilation time #[coroutine] move || { println!("{:?}", &b); diff --git a/tests/ui/lint/large_assignments/large_future.attribute.stderr b/tests/ui/lint/large_assignments/large_future.attribute.stderr index 734b7ff7ba22f..51e27c85cd258 100644 --- a/tests/ui/lint/large_assignments/large_future.attribute.stderr +++ b/tests/ui/lint/large_assignments/large_future.attribute.stderr @@ -1,5 +1,5 @@ -error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 +error: moving 10040 bytes + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -11,8 +11,8 @@ note: the lint level is defined here LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ -error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 +error: moving 10040 bytes + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.option-relocate.stderr b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr new file mode 100644 index 0000000000000..3c3a1fb3e5fb8 --- /dev/null +++ b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr @@ -0,0 +1,23 @@ +error: moving 10024 bytes + --> $DIR/large_future.rs:21:14 + | +LL | let z = (x, 42); + | ^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` +note: the lint level is defined here + --> $DIR/large_future.rs:1:9 + | +LL | #![deny(large_assignments)] + | ^^^^^^^^^^^^^^^^^ + +error: moving 10024 bytes + --> $DIR/large_future.rs:22:13 + | +LL | let a = z.0; + | ^^^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` + +error: aborting due to 2 previous errors + diff --git a/tests/ui/lint/large_assignments/large_future.option.stderr b/tests/ui/lint/large_assignments/large_future.option.stderr index 734b7ff7ba22f..51e27c85cd258 100644 --- a/tests/ui/lint/large_assignments/large_future.option.stderr +++ b/tests/ui/lint/large_assignments/large_future.option.stderr @@ -1,5 +1,5 @@ -error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 +error: moving 10040 bytes + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -11,8 +11,8 @@ note: the lint level is defined here LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ -error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 +error: moving 10040 bytes + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.rs b/tests/ui/lint/large_assignments/large_future.rs index 28c358bdbf086..43ae254663be9 100644 --- a/tests/ui/lint/large_assignments/large_future.rs +++ b/tests/ui/lint/large_assignments/large_future.rs @@ -3,8 +3,10 @@ #![cfg_attr(attribute, move_size_limit = "1000")] //@ build-fail //@ only-64bit -//@ revisions: attribute option -//@ [option]compile-flags: -Zmove-size-limit=1000 +//@ revisions: attribute option option-relocate +//@ compile-flags: -Zpack-coroutine-layout=no +//@ [option] compile-flags: -Zmove-size-limit=1000 +//@ [option-relocate] compile-flags: -Zmove-size-limit=1000 -Zpack-coroutine-layout=captures-only //@ edition:2018 //@ compile-flags: -Zmir-opt-level=0 diff --git a/tests/ui/mir/lint/storage-live.stderr b/tests/ui/mir/lint/storage-live.stderr index 50df9ae061fcc..fabf7f33745f0 100644 --- a/tests/ui/mir/lint/storage-live.stderr +++ b/tests/ui/mir/lint/storage-live.stderr @@ -1,4 +1,4 @@ -error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline) at bb0[1]: +error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline 1-1-000) at bb0[1]: StorageLive(_1) which already has storage here --> $DIR/storage-live.rs:21:13 | diff --git a/tests/ui/print_type_sizes/async.stdout b/tests/ui/print_type_sizes/async.classic.stdout similarity index 92% rename from tests/ui/print_type_sizes/async.stdout rename to tests/ui/print_type_sizes/async.classic.stdout index d3d6b6471c6ef..dcc0585a69576 100644 --- a/tests/ui/print_type_sizes/async.stdout +++ b/tests/ui/print_type_sizes/async.classic.stdout @@ -1,15 +1,14 @@ print-type-size type: `{async fn body of test()}`: 16386 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size upvar `.arg`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 16385 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size padding: 8192 bytes +print-type-size local `.__awaitee`: 1 bytes, alignment: 1 bytes, type: {async fn body of wait()} print-type-size local `.arg`: 8192 bytes -print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes print-type-size field `.value`: 8192 bytes print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/async.relocate.stdout b/tests/ui/print_type_sizes/async.relocate.stdout new file mode 100644 index 0000000000000..c6a7d8723e5c8 --- /dev/null +++ b/tests/ui/print_type_sizes/async.relocate.stdout @@ -0,0 +1,65 @@ +print-type-size type: `{async fn body of test()}`: 8194 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Suspend0`: 8193 bytes +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of wait()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/async.rs b/tests/ui/print_type_sizes/async.rs index b6ec88426345b..4020586b64f04 100644 --- a/tests/ui/print_type_sizes/async.rs +++ b/tests/ui/print_type_sizes/async.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type lib diff --git a/tests/ui/print_type_sizes/coroutine.classic.stdout b/tests/ui/print_type_sizes/coroutine.classic.stdout new file mode 100644 index 0000000000000..ee097da2fa240 --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.classic.stdout @@ -0,0 +1,15 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 16385 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size upvar `.array`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size variant `Returned`: 8192 bytes +print-type-size variant `Panicked`: 8192 bytes +print-type-size end padding: 8192 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.relocate.stdout b/tests/ui/print_type_sizes/coroutine.relocate.stdout new file mode 100644 index 0000000000000..79f5d093dc3ad --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.relocate.stdout @@ -0,0 +1,15 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 8193 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.array`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.rs b/tests/ui/print_type_sizes/coroutine.rs index 1533578878944..407d0873c015d 100644 --- a/tests/ui/print_type_sizes/coroutine.rs +++ b/tests/ui/print_type_sizes/coroutine.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z print-type-sizes --crate-type=lib //@ build-pass //@ ignore-pass diff --git a/tests/ui/print_type_sizes/coroutine.stdout b/tests/ui/print_type_sizes/coroutine.stdout deleted file mode 100644 index 339bbddfc2a93..0000000000000 --- a/tests/ui/print_type_sizes/coroutine.stdout +++ /dev/null @@ -1,10 +0,0 @@ -print-type-size type: `{coroutine@$DIR/coroutine.rs:11:5: 11:14}`: 8193 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Suspend0`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes diff --git a/tests/ui/rustc_public-ir-print/async-closure.stdout b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout similarity index 61% rename from tests/ui/rustc_public-ir-print/async-closure.stdout rename to tests/ui/rustc_public-ir-print/async-closure.classic.stdout index 5113dc5048b93..da3c1017c7e0d 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.stdout +++ b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout @@ -3,7 +3,7 @@ fn foo() -> () { let mut _0: (); let _1: i32; - let _2: {async closure@$DIR/async-closure.rs:9:13: 9:21}; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; let mut _3: &i32; debug y => _1; debug x => _2; @@ -13,7 +13,7 @@ fn foo() -> () { StorageLive(_2); StorageLive(_3); _3 = &_1; - _2 = {coroutine-closure@$DIR/async-closure.rs:9:13: 9:21}(move _3); + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); StorageDead(_3); _0 = (); StorageDead(_2); @@ -21,8 +21,8 @@ fn foo() -> () { return; } } -fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {async closure body@$DIR/async-closure.rs:9:22: 11:6} { - let mut _0: {async closure body@$DIR/async-closure.rs:9:22: 11:6}; +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; let mut _2: &i32; let mut _3: &i32; debug y => (*((*_1).0: &i32)); @@ -30,37 +30,37 @@ fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {a StorageLive(_2); _3 = CopyForDeref(((*_1).0: &i32)); _2 = &(*_3); - _0 = {coroutine@$DIR/async-closure.rs:9:22: 11:6}(move _2); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); StorageDead(_2); return; } } -fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: u32; - let mut _7: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _8: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _9: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _7: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _8: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _2; - debug y => (*((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})).0: &i32)); + debug y => (*(((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})) as variant#0).0: &i32)); debug y => _3; bb0: { - _7 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _7 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); _6 = discriminant((*_7)); switchInt(move _6) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { StorageLive(_3); - _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); - _4 = CopyForDeref(((*_8).0: &i32)); + _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _4 = CopyForDeref((((*_8) as variant#0).0: &i32)); _3 = (*_4); _5 = (); StorageDead(_3); _0 = std::task::Poll::Ready(move _5); - _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); discriminant((*_9)) = 1; return; } @@ -71,32 +71,32 @@ fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-clo unreachable; } } -fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: u32; - let mut _7: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _8: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; - let mut _9: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _7: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _8: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _2; - debug y => (*((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})).0: &i32)); + debug y => (*(((*(_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})) as variant#0).0: &i32)); debug y => _3; bb0: { - _7 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _7 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); _6 = discriminant((*_7)); switchInt(move _6) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { StorageLive(_3); - _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); - _4 = CopyForDeref(((*_8).0: &i32)); + _8 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _4 = CopyForDeref((((*_8) as variant#0).0: &i32)); _3 = (*_4); _5 = (); StorageDead(_3); _0 = std::task::Poll::Ready(move _5); - _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6})); + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); discriminant((*_9)) = 1; return; } diff --git a/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout new file mode 100644 index 0000000000000..09e8f8e831cef --- /dev/null +++ b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout @@ -0,0 +1,127 @@ +// WARNING: This is highly experimental output it's intended for rustc_public developers only. +// If you find a bug or want to improve the output open a issue at https://github.com/rust-lang/project-stable-mir. +fn foo() -> () { + let mut _0: (); + let _1: i32; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; + let mut _3: &i32; + debug y => _1; + debug x => _2; + bb0: { + StorageLive(_1); + _1 = 0_i32; + StorageLive(_2); + StorageLive(_3); + _3 = &_1; + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); + StorageDead(_3); + _0 = (); + StorageDead(_2); + StorageDead(_1); + return; + } +} +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _2: &i32; + let mut _3: &i32; + debug y => (*((*_1).0: &i32)); + bb0: { + StorageLive(_2); + _3 = CopyForDeref(((*_1).0: &i32)); + _2 = &(*_3); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); + StorageDead(_2); + return; + } +} +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let mut _4: &i32; + let _5: &i32; + let _6: &i32; + let mut _7: (); + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _10: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _11: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _2; + debug y => (*_5); + debug y => _3; + debug y => _5; + bb0: { + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + StorageLive(_5); + StorageLive(_6); + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _6 = move (((*_10) as variant#0).0: &i32); + _5 = move _6; + StorageDead(_6); + StorageLive(_3); + _4 = CopyForDeref(_5); + _3 = (*_4); + _7 = (); + StorageDead(_3); + StorageDead(_5); + _0 = std::task::Poll::Ready(move _7); + _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + discriminant((*_11)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let mut _4: &i32; + let _5: &i32; + let _6: &i32; + let mut _7: (); + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _10: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _11: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _2; + debug y => (*_5); + debug y => _3; + debug y => _5; + bb0: { + _9 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + StorageLive(_5); + StorageLive(_6); + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _6 = move (((*_10) as variant#0).0: &i32); + _5 = move _6; + StorageDead(_6); + StorageLive(_3); + _4 = CopyForDeref(_5); + _3 = (*_4); + _7 = (); + StorageDead(_3); + StorageDead(_5); + _0 = std::task::Poll::Ready(move _7); + _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + discriminant((*_11)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} diff --git a/tests/ui/rustc_public-ir-print/async-closure.rs b/tests/ui/rustc_public-ir-print/async-closure.rs index bd8c7e888a37c..4457468752b2e 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.rs +++ b/tests/ui/rustc_public-ir-print/async-closure.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z unpretty=stable-mir --crate-type lib -C panic=abort -Zmir-opt-level=0 //@ check-pass //@ only-64bit From f13598990dcbe1d93b71d06ab7f6dc2a280bd505 Mon Sep 17 00:00:00 2001 From: Xiangfei Ding Date: Fri, 26 Sep 2025 05:09:33 +0800 Subject: [PATCH 2/2] make sure that retagging body arguments stays the first instruction --- .../src/coroutine/relocate_upvars.rs | 56 +++++++++++++---- ...re#0}.RelocateUpvars.after.panic-abort.mir | 54 +++++++++-------- ...e#0}.RelocateUpvars.after.panic-unwind.mir | 60 ++++++++++--------- ...e#0}.RelocateUpvars.before.panic-abort.mir | 8 ++- ...#0}.RelocateUpvars.before.panic-unwind.mir | 8 ++- .../async-closure.relocate.stdout | 16 ++--- 6 files changed, 125 insertions(+), 77 deletions(-) diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs index 96fece430410d..f7f0eaf3ef01a 100644 --- a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -102,6 +102,7 @@ use std::borrow::Cow; +use itertools::Itertools; use rustc_abi::FieldIdx; use rustc_index::bit_set::DenseBitSet; use rustc_index::{IndexSlice, IndexVec}; @@ -261,6 +262,28 @@ impl RelocateUpvars { | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), }; + // HACK: in case `AddRetag` is already run, we have one `Retag` at the body entrance + // so we need to make sure that first `Retag` is run. + let last_retag = body.basic_blocks[START_BLOCK] + .statements + .iter() + .find_position(|stmt| !matches!(stmt.kind, StatementKind::Retag(_, _))); + let retagged_start_block = if let Some((index, _)) = last_retag { + let span = body.span; + let bbs = body.basic_blocks_mut(); + let stmts = bbs[START_BLOCK].statements.split_off(index); + let terminator = bbs[START_BLOCK].terminator.take(); + let split_start_block = bbs.push(BasicBlockData::new_stmts(stmts, terminator, false)); + bbs[START_BLOCK].statements.shrink_to_fit(); + bbs[START_BLOCK].terminator = Some(Terminator { + kind: TerminatorKind::Goto { target: split_start_block }, + source_info: SourceInfo::outermost(span), + }); + Some(split_start_block) + } else { + None + }; + if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { mir_dumper.set_disambiguator(&"before").dump_mir(body); } @@ -308,7 +331,7 @@ impl RelocateUpvars { SubstituteUpvarVisitor { tcx, mappings: &substitution_mapping }.visit_body(body); rewrite_drop_coroutine_struct(tcx, body, &substitution_mapping); - insert_substitution_prologue(body, &substitution_mapping); + insert_substitution_prologue(body, retagged_start_block, &substitution_mapping); patch_missing_storage_deads(tcx, body, &substitution_mapping); hydrate_var_debug_info(body, &substitution_mapping); if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { @@ -445,10 +468,11 @@ fn rewrite_drop_coroutine_struct<'tcx>( fn insert_substitution_prologue<'tcx>( body: &mut Body<'tcx>, + retagged_start_block: Option, substitution_mapping: &IndexSlice>, ) { let mut patch = MirPatch::new(body); - let mut stmts = Vec::with_capacity(2 * substitution_mapping.len()); + let mut stmts = Vec::with_capacity(5 * substitution_mapping.len()); for &UpvarSubstitution { local, reloc, upvar_place, span, name: _ } in substitution_mapping { // For each upvar-local _$i let source_info = SourceInfo::outermost(span); @@ -481,22 +505,30 @@ fn insert_substitution_prologue<'tcx>( let source_info = SourceInfo::outermost(body.span); let prologue = patch.new_block(BasicBlockData::new_stmts( stmts, - Some(Terminator { source_info, kind: TerminatorKind::Goto { target: START_BLOCK } }), + Some(Terminator { + source_info, + kind: TerminatorKind::Goto { target: retagged_start_block.unwrap_or(START_BLOCK) }, + }), false, )); patch.apply(body); // Manually patch so that prologue is the new entry-point - let preds = body.basic_blocks.predecessors()[START_BLOCK].clone(); - let basic_blocks = body.basic_blocks.as_mut(); - for pred in preds { - basic_blocks[pred].terminator_mut().successors_mut(|target| { - if *target == START_BLOCK { - *target = prologue; - } - }); + if retagged_start_block.is_some() { + let basic_blocks = body.basic_blocks.as_mut(); + basic_blocks[START_BLOCK].terminator_mut().successors_mut(|target| *target = prologue); + } else { + let preds = body.basic_blocks.predecessors()[START_BLOCK].clone(); + let basic_blocks = body.basic_blocks.as_mut(); + for pred in preds { + basic_blocks[pred].terminator_mut().successors_mut(|target| { + if *target == START_BLOCK { + *target = prologue; + } + }); + } + basic_blocks.swap(START_BLOCK, prologue); } - basic_blocks.swap(START_BLOCK, prologue); } /// Occasionally there are upvar locals left without `StorageDead` because diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir index d6067ec2ba63f..223dead5ce871 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-abort.mir @@ -16,12 +16,7 @@ yields () let _10: &mut std::string::String; bb0: { - StorageLive(_9); - StorageLive(_10); - _10 = move (_1.0: &mut std::string::String); - _9 = move _10; - StorageDead(_10); - goto -> bb18; + goto -> bb19; } bb1: { @@ -47,7 +42,7 @@ yields () StorageDead(_5); StorageDead(_4); _0 = const (); - goto -> bb13; + goto -> bb14; } bb5: { @@ -61,55 +56,64 @@ yields () } bb7: { - goto -> bb17; + goto -> bb18; } bb8: { coroutine_drop; } - bb9 (cleanup): { - unreachable; + bb9: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind unreachable]; } bb10 (cleanup): { - StorageDead(_9); - goto -> bb9; + unreachable; } bb11 (cleanup): { + StorageDead(_9); goto -> bb10; } - bb12: { + bb12 (cleanup): { + goto -> bb11; + } + + bb13: { StorageDead(_9); goto -> bb5; } - bb13: { - goto -> bb12; + bb14: { + goto -> bb13; } - bb14 (cleanup): { + bb15 (cleanup): { StorageDead(_9); - goto -> bb9; + goto -> bb10; } - bb15 (cleanup): { - goto -> bb14; + bb16 (cleanup): { + goto -> bb15; } - bb16: { + bb17: { StorageDead(_9); goto -> bb8; } - bb17: { - goto -> bb16; + bb18: { + goto -> bb17; } - bb18: { - StorageLive(_3); - _3 = String::new() -> [return: bb1, unwind unreachable]; + bb19: { + StorageLive(_9); + StorageLive(_10); + _10 = move (_1.0: &mut std::string::String); + _9 = move _10; + StorageDead(_10); + goto -> bb9; } } diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir index 7c90429ef8dc6..2f0c930d99717 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.after.panic-unwind.mir @@ -16,12 +16,7 @@ yields () let _10: &mut std::string::String; bb0: { - StorageLive(_9); - StorageLive(_10); - _10 = move (_1.0: &mut std::string::String); - _9 = move _10; - StorageDead(_10); - goto -> bb23; + goto -> bb24; } bb1: { @@ -53,7 +48,7 @@ yields () StorageDead(_5); StorageDead(_4); _0 = const (); - goto -> bb15; + goto -> bb16; } bb6: { @@ -67,7 +62,7 @@ yields () } bb8: { - goto -> bb19; + goto -> bb20; } bb9: { @@ -76,64 +71,73 @@ yields () bb10 (cleanup): { StorageDead(_3); - goto -> bb22; + goto -> bb23; } bb11 (cleanup): { resume; } - bb12 (cleanup): { + bb12: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb10]; + } + + bb13 (cleanup): { StorageDead(_9); goto -> bb11; } - bb13 (cleanup): { - goto -> bb12; + bb14 (cleanup): { + goto -> bb13; } - bb14: { + bb15: { StorageDead(_9); goto -> bb6; } - bb15: { - goto -> bb14; + bb16: { + goto -> bb15; } - bb16 (cleanup): { + bb17 (cleanup): { StorageDead(_9); goto -> bb11; } - bb17 (cleanup): { - goto -> bb16; + bb18 (cleanup): { + goto -> bb17; } - bb18: { + bb19: { StorageDead(_9); goto -> bb9; } - bb19: { - goto -> bb18; + bb20: { + goto -> bb19; } - bb20 (cleanup): { + bb21 (cleanup): { terminate(cleanup); } - bb21 (cleanup): { + bb22 (cleanup): { StorageDead(_9); goto -> bb11; } - bb22 (cleanup): { - goto -> bb21; + bb23 (cleanup): { + goto -> bb22; } - bb23: { - StorageLive(_3); - _3 = String::new() -> [return: bb1, unwind: bb10]; + bb24: { + StorageLive(_9); + StorageLive(_10); + _10 = move (_1.0: &mut std::string::String); + _9 = move _10; + StorageDead(_10); + goto -> bb12; } } diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir index 2fd8713094c05..b31d3fbfdcb9b 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-abort.mir @@ -13,8 +13,7 @@ yields () let mut _8: &mut std::string::String; bb0: { - StorageLive(_3); - _3 = String::new() -> [return: bb1, unwind unreachable]; + goto -> bb9; } bb1: { @@ -60,4 +59,9 @@ yields () bb8: { coroutine_drop; } + + bb9: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind unreachable]; + } } diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir index eb4762a233a9f..4d4555326e568 100644 --- a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.before.panic-unwind.mir @@ -13,8 +13,7 @@ yields () let mut _8: &mut std::string::String; bb0: { - StorageLive(_3); - _3 = String::new() -> [return: bb1, unwind: bb10]; + goto -> bb12; } bb1: { @@ -75,4 +74,9 @@ yields () bb11 (cleanup): { resume; } + + bb12: { + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb10]; + } } diff --git a/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout index 09e8f8e831cef..dcd79abae9298 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout +++ b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout @@ -58,8 +58,8 @@ fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-clo bb1: { StorageLive(_5); StorageLive(_6); - _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); - _6 = move (((*_10) as variant#0).0: &i32); + _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _6 = move (((*_11) as variant#0).0: &i32); _5 = move _6; StorageDead(_6); StorageLive(_3); @@ -69,8 +69,8 @@ fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-clo StorageDead(_3); StorageDead(_5); _0 = std::task::Poll::Ready(move _7); - _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); - discriminant((*_11)) = 1; + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + discriminant((*_10)) = 1; return; } bb2: { @@ -103,8 +103,8 @@ fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-c bb1: { StorageLive(_5); StorageLive(_6); - _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); - _6 = move (((*_10) as variant#0).0: &i32); + _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + _6 = move (((*_11) as variant#0).0: &i32); _5 = move _6; StorageDead(_6); StorageLive(_3); @@ -114,8 +114,8 @@ fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-c StorageDead(_3); StorageDead(_5); _0 = std::task::Poll::Ready(move _7); - _11 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); - discriminant((*_11)) = 1; + _10 = CopyForDeref((_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6})); + discriminant((*_10)) = 1; return; } bb2: {