58
58
//! borrowing from the outer closure, and we simply peel off a `deref` projection
59
59
//! from them. This second body is stored alongside the first body, and optimized
60
60
//! with it in lockstep. When we need to resolve a body for `FnOnce` or `AsyncFnOnce`,
61
- //! we use this "by move" body instead.
61
+ //! we use this "by-move" body instead.
62
+ //!
63
+ //! ## How does this work?
64
+ //!
65
+ //! This pass essentially remaps the body of the (child) closure of the coroutine-closure
66
+ //! to take the set of upvars of the parent closure by value. This at least requires
67
+ //! changing a by-ref upvar to be by-value in the case that the outer coroutine-closure
68
+ //! captures something by value; however, it may also require renumbering field indices
69
+ //! in case precise captures (edition 2021 closure capture rules) caused the inner coroutine
70
+ //! to split one field capture into two.
62
71
63
72
use rustc_data_structures:: unord:: UnordMap ;
64
73
use rustc_hir as hir;
@@ -117,8 +126,15 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
117
126
118
127
let mut field_remapping = UnordMap :: default ( ) ;
119
128
129
+ // One parent capture may correspond to several child captures if we end up
130
+ // refining the set of captures via edition-2021 precise captures. We want to
131
+ // match up any number of child captures with one parent capture, so we keep
132
+ // peeking off this `Peekable` until the child doesn't match anymore.
120
133
let mut parent_captures =
121
134
tcx. closure_captures ( parent_def_id) . iter ( ) . copied ( ) . enumerate ( ) . peekable ( ) ;
135
+ // Make sure we use every field at least once, b/c why are we capturing something
136
+ // if it's not used in the inner coroutine.
137
+ let mut field_used_at_least_once = false ;
122
138
123
139
for ( child_field_idx, child_capture) in tcx
124
140
. closure_captures ( coroutine_def_id)
@@ -133,20 +149,36 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
133
149
bug ! ( "we ran out of parent captures!" )
134
150
} ;
135
151
152
+ assert ! (
153
+ child_capture. place. projections. len( ) >= parent_capture. place. projections. len( )
154
+ ) ;
155
+ // A parent matches a child they share the same prefix of projections.
156
+ // The child may have more, if it is capturing sub-fields out of
157
+ // something that is captured by-move in the parent closure.
136
158
if !std:: iter:: zip (
137
159
& child_capture. place . projections ,
138
160
& parent_capture. place . projections ,
139
161
)
140
162
. all ( |( child, parent) | child. kind == parent. kind )
141
163
{
164
+ // Make sure the field was used at least once.
165
+ assert ! (
166
+ field_used_at_least_once,
167
+ "we captured {parent_capture:#?} but it was not used in the child coroutine?"
168
+ ) ;
169
+ field_used_at_least_once = false ;
142
170
// Skip this field.
143
171
let _ = parent_captures. next ( ) . unwrap ( ) ;
144
172
continue ;
145
173
}
146
174
175
+ // Store this set of additional projections (fields and derefs).
176
+ // We need to re-apply them later.
147
177
let child_precise_captures =
148
178
& child_capture. place . projections [ parent_capture. place . projections . len ( ) ..] ;
149
179
180
+ // If the parent captures by-move, and the child captures by-ref, then we
181
+ // need to peel an additional `deref` off of the body of the child.
150
182
let needs_deref = child_capture. is_by_ref ( ) && !parent_capture. is_by_ref ( ) ;
151
183
if needs_deref {
152
184
assert_ne ! (
@@ -157,6 +189,8 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
157
189
) ;
158
190
}
159
191
192
+ // Finally, store the type of the parent's captured place. We need
193
+ // this when building the field projection in the MIR body later on.
160
194
let mut parent_capture_ty = parent_capture. place . ty ( ) ;
161
195
parent_capture_ty = match parent_capture. info . capture_kind {
162
196
ty:: UpvarCapture :: ByValue => parent_capture_ty,
@@ -178,6 +212,7 @@ impl<'tcx> MirPass<'tcx> for ByMoveBody {
178
212
) ,
179
213
) ;
180
214
215
+ field_used_at_least_once = true ;
181
216
break ;
182
217
}
183
218
}
@@ -226,21 +261,34 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> {
226
261
context : mir:: visit:: PlaceContext ,
227
262
location : mir:: Location ,
228
263
) {
264
+ // Initializing an upvar local always starts with `CAPTURE_STRUCT_LOCAL` and a
265
+ // field projection. If this is in `field_remapping`, then it must not be an
266
+ // arg from calling the closure, but instead an upvar.
229
267
if place. local == ty:: CAPTURE_STRUCT_LOCAL
230
268
&& let Some ( ( & mir:: ProjectionElem :: Field ( idx, _) , projection) ) =
231
269
place. projection . split_first ( )
232
270
&& let Some ( & ( remapped_idx, remapped_ty, needs_deref, additional_projections) ) =
233
271
self . field_remapping . get ( & idx)
234
272
{
273
+ // As noted before, if the parent closure captures a field by value, and
274
+ // the child captures a field by ref, then for the by-move body we're
275
+ // generating, we also are taking that field by value. Peel off a deref,
276
+ // since a layer of reffing has now become redundant.
235
277
let final_deref = if needs_deref {
236
- let Some ( ( mir:: ProjectionElem :: Deref , rest ) ) = projection. split_first ( ) else {
237
- bug ! ( ) ;
278
+ let [ mir:: ProjectionElem :: Deref ] = projection else {
279
+ bug ! ( "There should only be a single deref for an upvar local initialization" ) ;
238
280
} ;
239
- rest
281
+ & [ ]
240
282
} else {
241
283
projection
242
284
} ;
243
285
286
+ // The only thing that should be left is a deref, if the parent captured
287
+ // an upvar by-ref.
288
+ std:: assert_matches:: assert_matches!( final_deref, [ ] | [ mir:: ProjectionElem :: Deref ] ) ;
289
+
290
+ // For all of the additional projections that come out of precise capturing,
291
+ // re-apply these projections.
244
292
let additional_projections =
245
293
additional_projections. iter ( ) . map ( |elem| match elem. kind {
246
294
ProjectionKind :: Deref => mir:: ProjectionElem :: Deref ,
@@ -250,6 +298,10 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> {
250
298
_ => unreachable ! ( "precise captures only through fields and derefs" ) ,
251
299
} ) ;
252
300
301
+ // We start out with an adjusted field index (and ty), representing the
302
+ // upvar that we get from our parent closure. We apply any of the additional
303
+ // projections to make sure that to the rest of the body of the closure, the
304
+ // place looks the same, and then apply that final deref if necessary.
253
305
* place = mir:: Place {
254
306
local : place. local ,
255
307
projection : self . tcx . mk_place_elems_from_iter (
0 commit comments