|
52 | 52 |
|
53 | 53 | mod by_move_body; |
54 | 54 | mod drop; |
55 | | -use std::{iter, ops}; |
| 55 | +use std::ops; |
56 | 56 |
|
57 | 57 | pub(super) use by_move_body::coroutine_by_move_body_def_id; |
58 | 58 | use drop::{ |
59 | 59 | cleanup_async_drops, create_coroutine_drop_shim, create_coroutine_drop_shim_async, |
60 | 60 | create_coroutine_drop_shim_proxy_async, elaborate_coroutine_drops, expand_async_drops, |
61 | 61 | has_expandable_async_drops, insert_clean_drop, |
62 | 62 | }; |
| 63 | +use itertools::izip; |
63 | 64 | use rustc_abi::{FieldIdx, VariantIdx}; |
64 | 65 | use rustc_data_structures::fx::FxHashSet; |
65 | 66 | use rustc_errors::pluralize; |
@@ -752,53 +753,53 @@ fn locals_live_across_suspend_points<'tcx>( |
752 | 753 | let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len()); |
753 | 754 |
|
754 | 755 | for (block, data) in body.basic_blocks.iter_enumerated() { |
755 | | - if let TerminatorKind::Yield { .. } = data.terminator().kind { |
756 | | - let loc = Location { block, statement_index: data.statements.len() }; |
757 | | - |
758 | | - liveness.seek_to_block_end(block); |
759 | | - let mut live_locals = liveness.get().clone(); |
760 | | - |
761 | | - if !movable { |
762 | | - // The `liveness` variable contains the liveness of MIR locals ignoring borrows. |
763 | | - // This is correct for movable coroutines since borrows cannot live across |
764 | | - // suspension points. However for immovable coroutines we need to account for |
765 | | - // borrows, so we conservatively assume that all borrowed locals are live until |
766 | | - // we find a StorageDead statement referencing the locals. |
767 | | - // To do this we just union our `liveness` result with `borrowed_locals`, which |
768 | | - // contains all the locals which has been borrowed before this suspension point. |
769 | | - // If a borrow is converted to a raw reference, we must also assume that it lives |
770 | | - // forever. Note that the final liveness is still bounded by the storage liveness |
771 | | - // of the local, which happens using the `intersect` operation below. |
772 | | - borrowed_locals_cursor2.seek_before_primary_effect(loc); |
773 | | - live_locals.union(borrowed_locals_cursor2.get()); |
774 | | - } |
| 756 | + let TerminatorKind::Yield { .. } = data.terminator().kind else { continue }; |
| 757 | + |
| 758 | + let loc = Location { block, statement_index: data.statements.len() }; |
| 759 | + |
| 760 | + liveness.seek_to_block_end(block); |
| 761 | + let mut live_locals = liveness.get().clone(); |
| 762 | + |
| 763 | + if !movable { |
| 764 | + // The `liveness` variable contains the liveness of MIR locals ignoring borrows. |
| 765 | + // This is correct for movable coroutines since borrows cannot live across |
| 766 | + // suspension points. However for immovable coroutines we need to account for |
| 767 | + // borrows, so we conservatively assume that all borrowed locals are live until |
| 768 | + // we find a StorageDead statement referencing the locals. |
| 769 | + // To do this we just union our `liveness` result with `borrowed_locals`, which |
| 770 | + // contains all the locals which has been borrowed before this suspension point. |
| 771 | + // If a borrow is converted to a raw reference, we must also assume that it lives |
| 772 | + // forever. Note that the final liveness is still bounded by the storage liveness |
| 773 | + // of the local, which happens using the `intersect` operation below. |
| 774 | + borrowed_locals_cursor2.seek_before_primary_effect(loc); |
| 775 | + live_locals.union(borrowed_locals_cursor2.get()); |
| 776 | + } |
775 | 777 |
|
776 | | - // Store the storage liveness for later use so we can restore the state |
777 | | - // after a suspension point |
778 | | - storage_live.seek_before_primary_effect(loc); |
779 | | - storage_liveness_map[block] = Some(storage_live.get().clone()); |
| 778 | + // Store the storage liveness for later use so we can restore the state |
| 779 | + // after a suspension point |
| 780 | + storage_live.seek_before_primary_effect(loc); |
| 781 | + storage_liveness_map[block] = Some(storage_live.get().clone()); |
780 | 782 |
|
781 | | - // Locals live are live at this point only if they are used across |
782 | | - // suspension points (the `liveness` variable) |
783 | | - // and their storage is required (the `storage_required` variable) |
784 | | - requires_storage_cursor.seek_before_primary_effect(loc); |
785 | | - live_locals.intersect(requires_storage_cursor.get()); |
| 783 | + // Locals live are live at this point only if they are used across |
| 784 | + // suspension points (the `liveness` variable) |
| 785 | + // and their storage is required (the `storage_required` variable) |
| 786 | + requires_storage_cursor.seek_before_primary_effect(loc); |
| 787 | + live_locals.intersect(requires_storage_cursor.get()); |
786 | 788 |
|
787 | | - // The coroutine argument is ignored. |
788 | | - live_locals.remove(SELF_ARG); |
| 789 | + // The coroutine argument is ignored. |
| 790 | + live_locals.remove(SELF_ARG); |
789 | 791 |
|
790 | | - debug!("loc = {:?}, live_locals = {:?}", loc, live_locals); |
| 792 | + debug!(?loc, ?live_locals); |
791 | 793 |
|
792 | | - // Add the locals live at this suspension point to the set of locals which live across |
793 | | - // any suspension points |
794 | | - live_locals_at_any_suspension_point.union(&live_locals); |
| 794 | + // Add the locals live at this suspension point to the set of locals which live across |
| 795 | + // any suspension points |
| 796 | + live_locals_at_any_suspension_point.union(&live_locals); |
795 | 797 |
|
796 | | - live_locals_at_suspension_points.push(live_locals); |
797 | | - source_info_at_suspension_points.push(data.terminator().source_info); |
798 | | - } |
| 798 | + live_locals_at_suspension_points.push(live_locals); |
| 799 | + source_info_at_suspension_points.push(data.terminator().source_info); |
799 | 800 | } |
800 | 801 |
|
801 | | - debug!("live_locals_anywhere = {:?}", live_locals_at_any_suspension_point); |
| 802 | + debug!(?live_locals_at_any_suspension_point); |
802 | 803 | let saved_locals = CoroutineSavedLocals(live_locals_at_any_suspension_point); |
803 | 804 |
|
804 | 805 | // Renumber our liveness_map bitsets to include only the locals we are |
@@ -999,8 +1000,8 @@ fn compute_layout<'tcx>( |
999 | 1000 | } = liveness; |
1000 | 1001 |
|
1001 | 1002 | // Gather live local types and their indices. |
1002 | | - let mut locals = IndexVec::<CoroutineSavedLocal, _>::new(); |
1003 | | - let mut tys = IndexVec::<CoroutineSavedLocal, _>::new(); |
| 1003 | + let mut locals = IndexVec::<CoroutineSavedLocal, _>::with_capacity(saved_locals.domain_size()); |
| 1004 | + let mut tys = IndexVec::<CoroutineSavedLocal, _>::with_capacity(saved_locals.domain_size()); |
1004 | 1005 | for (saved_local, local) in saved_locals.iter_enumerated() { |
1005 | 1006 | debug!("coroutine saved local {:?} => {:?}", saved_local, local); |
1006 | 1007 |
|
@@ -1034,38 +1035,39 @@ fn compute_layout<'tcx>( |
1034 | 1035 | // In debuginfo, these will correspond to the beginning (UNRESUMED) or end |
1035 | 1036 | // (RETURNED, POISONED) of the function. |
1036 | 1037 | let body_span = body.source_scopes[OUTERMOST_SOURCE_SCOPE].span; |
1037 | | - let mut variant_source_info: IndexVec<VariantIdx, SourceInfo> = [ |
| 1038 | + let mut variant_source_info: IndexVec<VariantIdx, SourceInfo> = IndexVec::with_capacity( |
| 1039 | + CoroutineArgs::RESERVED_VARIANTS + live_locals_at_suspension_points.len(), |
| 1040 | + ); |
| 1041 | + variant_source_info.extend([ |
1038 | 1042 | SourceInfo::outermost(body_span.shrink_to_lo()), |
1039 | 1043 | SourceInfo::outermost(body_span.shrink_to_hi()), |
1040 | 1044 | SourceInfo::outermost(body_span.shrink_to_hi()), |
1041 | | - ] |
1042 | | - .iter() |
1043 | | - .copied() |
1044 | | - .collect(); |
| 1045 | + ]); |
1045 | 1046 |
|
1046 | 1047 | // Build the coroutine variant field list. |
1047 | 1048 | // Create a map from local indices to coroutine struct indices. |
1048 | | - let mut variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, CoroutineSavedLocal>> = |
1049 | | - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); |
| 1049 | + let mut variant_fields: IndexVec<VariantIdx, _> = IndexVec::from_elem_n( |
| 1050 | + IndexVec::new(), |
| 1051 | + CoroutineArgs::RESERVED_VARIANTS + live_locals_at_suspension_points.len(), |
| 1052 | + ); |
1050 | 1053 | let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); |
1051 | | - for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { |
1052 | | - let variant_index = |
1053 | | - VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); |
1054 | | - let mut fields = IndexVec::new(); |
1055 | | - for (idx, saved_local) in live_locals.iter().enumerate() { |
1056 | | - fields.push(saved_local); |
| 1054 | + for (live_locals, &source_info_at_suspension_point, (variant_index, fields)) in izip!( |
| 1055 | + &live_locals_at_suspension_points, |
| 1056 | + &source_info_at_suspension_points, |
| 1057 | + variant_fields.iter_enumerated_mut().skip(CoroutineArgs::RESERVED_VARIANTS) |
| 1058 | + ) { |
| 1059 | + *fields = live_locals.iter().collect(); |
| 1060 | + for (idx, &saved_local) in fields.iter_enumerated() { |
1057 | 1061 | // Note that if a field is included in multiple variants, we will |
1058 | 1062 | // just use the first one here. That's fine; fields do not move |
1059 | 1063 | // around inside coroutines, so it doesn't matter which variant |
1060 | 1064 | // index we access them by. |
1061 | | - let idx = FieldIdx::from_usize(idx); |
1062 | 1065 | remap[locals[saved_local]] = Some((tys[saved_local].ty, variant_index, idx)); |
1063 | 1066 | } |
1064 | | - variant_fields.push(fields); |
1065 | | - variant_source_info.push(source_info_at_suspension_points[suspension_point_idx]); |
| 1067 | + variant_source_info.push(source_info_at_suspension_point); |
1066 | 1068 | } |
1067 | | - debug!("coroutine variant_fields = {:?}", variant_fields); |
1068 | | - debug!("coroutine storage_conflicts = {:#?}", storage_conflicts); |
| 1069 | + debug!(?variant_fields); |
| 1070 | + debug!(?storage_conflicts); |
1069 | 1071 |
|
1070 | 1072 | let mut field_names = IndexVec::from_elem(None, &tys); |
1071 | 1073 | for var in &body.var_debug_info { |
|
0 commit comments