@@ -93,6 +93,7 @@ fn escaping_locals<'tcx>(
93
93
set. insert_range ( RETURN_PLACE ..=Local :: from_usize ( body. arg_count ) ) ;
94
94
for ( local, decl) in body. local_decls ( ) . iter_enumerated ( ) {
95
95
if excluded. contains ( local) || is_excluded_ty ( decl. ty ) {
96
+ trace ! ( ?local, "early exclusion" ) ;
96
97
set. insert ( local) ;
97
98
}
98
99
}
@@ -105,13 +106,17 @@ fn escaping_locals<'tcx>(
105
106
}
106
107
107
108
impl < ' tcx > Visitor < ' tcx > for EscapeVisitor {
108
- fn visit_local ( & mut self , local : Local , _: PlaceContext , _: Location ) {
109
- self . set . insert ( local) ;
109
+ fn visit_local ( & mut self , local : Local , _: PlaceContext , loc : Location ) {
110
+ if self . set . insert ( local) {
111
+ trace ! ( ?local, "escapes at {loc:?}" ) ;
112
+ }
110
113
}
111
114
112
115
fn visit_place ( & mut self , place : & Place < ' tcx > , context : PlaceContext , location : Location ) {
113
116
// Mirror the implementation in PreFlattenVisitor.
114
- if let & [ PlaceElem :: Field ( ..) , ..] = & place. projection [ ..] {
117
+ if let & [ PlaceElem :: Field ( ..) | PlaceElem :: ConstantIndex { from_end : false , .. } , ..] =
118
+ & place. projection [ ..]
119
+ {
115
120
return ;
116
121
}
117
122
self . super_place ( place, context, location) ;
@@ -126,7 +131,7 @@ fn escaping_locals<'tcx>(
126
131
if lvalue. as_local ( ) . is_some ( ) {
127
132
match rvalue {
128
133
// Aggregate assignments are expanded in run_pass.
129
- Rvalue :: Aggregate ( ..) | Rvalue :: Use ( ..) => {
134
+ Rvalue :: Repeat ( .. ) | Rvalue :: Aggregate ( ..) | Rvalue :: Use ( ..) => {
130
135
self . visit_rvalue ( rvalue, location) ;
131
136
return ;
132
137
}
@@ -152,32 +157,62 @@ fn escaping_locals<'tcx>(
152
157
}
153
158
}
154
159
160
+ #[ derive( Copy , Clone , Debug ) ]
161
+ enum LocalMode < ' tcx > {
162
+ Field ( Ty < ' tcx > , Local ) ,
163
+ Index ( Local ) ,
164
+ }
165
+
166
+ impl < ' tcx > LocalMode < ' tcx > {
167
+ fn local ( self ) -> Local {
168
+ match self {
169
+ LocalMode :: Field ( _, l) | LocalMode :: Index ( l) => l,
170
+ }
171
+ }
172
+
173
+ fn elem ( self , field : FieldIdx ) -> PlaceElem < ' tcx > {
174
+ match self {
175
+ LocalMode :: Field ( ty, _) => PlaceElem :: Field ( field, ty) ,
176
+ LocalMode :: Index ( _) => PlaceElem :: ConstantIndex {
177
+ offset : field. as_u32 ( ) as u64 ,
178
+ min_length : field. as_u32 ( ) as u64 + 1 ,
179
+ from_end : false ,
180
+ } ,
181
+ }
182
+ }
183
+ }
184
+
155
185
#[ derive( Default , Debug ) ]
156
186
struct ReplacementMap < ' tcx > {
157
187
/// Pre-computed list of all "new" locals for each "old" local. This is used to expand storage
158
188
/// and deinit statement and debuginfo.
159
- fragments : IndexVec < Local , Option < IndexVec < FieldIdx , Option < ( Ty < ' tcx > , Local ) > > > > ,
189
+ fragments : IndexVec < Local , Option < IndexVec < FieldIdx , Option < LocalMode < ' tcx > > > > > ,
160
190
}
161
191
162
192
impl < ' tcx > ReplacementMap < ' tcx > {
163
193
fn replace_place ( & self , tcx : TyCtxt < ' tcx > , place : PlaceRef < ' tcx > ) -> Option < Place < ' tcx > > {
164
- let & [ PlaceElem :: Field ( f , _ ) , ref rest @ ..] = place. projection else {
194
+ let & [ first , ref rest @ ..] = place. projection else {
165
195
return None ;
166
196
} ;
197
+ let f = match first {
198
+ PlaceElem :: Field ( f, _) => f,
199
+ PlaceElem :: ConstantIndex { offset, .. } => FieldIdx :: from_u32 ( offset. try_into ( ) . ok ( ) ?) ,
200
+ _ => return None ,
201
+ } ;
167
202
let fields = self . fragments [ place. local ] . as_ref ( ) ?;
168
- let ( _ , new_local) = fields[ f] ?;
203
+ let new_local = fields[ f] ?. local ( ) ;
169
204
Some ( Place { local : new_local, projection : tcx. mk_place_elems ( rest) } )
170
205
}
171
206
172
207
fn place_fragments (
173
208
& self ,
174
209
place : Place < ' tcx > ,
175
- ) -> Option < impl Iterator < Item = ( FieldIdx , Ty < ' tcx > , Local ) > + ' _ > {
210
+ ) -> Option < impl Iterator < Item = ( FieldIdx , LocalMode < ' tcx > ) > + ' _ > {
176
211
let local = place. as_local ( ) ?;
177
212
let fields = self . fragments [ local] . as_ref ( ) ?;
178
- Some ( fields. iter_enumerated ( ) . filter_map ( |( field, & opt_ty_local ) | {
179
- let ( ty , local) = opt_ty_local ?;
180
- Some ( ( field, ty , local) )
213
+ Some ( fields. iter_enumerated ( ) . filter_map ( |( field, & local ) | {
214
+ let local = local ?;
215
+ Some ( ( field, local) )
181
216
} ) )
182
217
}
183
218
}
@@ -200,15 +235,32 @@ fn compute_flattening<'tcx>(
200
235
}
201
236
let decl = body. local_decls [ local] . clone ( ) ;
202
237
let ty = decl. ty ;
203
- iter_fields ( ty, tcx, param_env, |variant, field, field_ty| {
204
- if variant. is_some ( ) {
205
- // Downcasts are currently not supported.
206
- return ;
207
- } ;
208
- let new_local =
209
- body. local_decls . push ( LocalDecl { ty : field_ty, user_ty : None , ..decl. clone ( ) } ) ;
210
- fragments. get_or_insert_with ( local, IndexVec :: new) . insert ( field, ( field_ty, new_local) ) ;
211
- } ) ;
238
+ if let ty:: Array ( inner, count) = ty. kind ( )
239
+ && let Some ( count) = count. try_eval_target_usize ( tcx, param_env)
240
+ && count <= 64
241
+ {
242
+ let fragments = fragments. get_or_insert_with ( local, IndexVec :: new) ;
243
+ for field in 0 ..( count as u32 ) {
244
+ let new_local =
245
+ body. local_decls . push ( LocalDecl { ty : * inner, user_ty : None , ..decl. clone ( ) } ) ;
246
+ fragments. insert ( FieldIdx :: from_u32 ( field) , LocalMode :: Index ( new_local) ) ;
247
+ }
248
+ } else {
249
+ iter_fields ( ty, tcx, param_env, |variant, field, field_ty| {
250
+ if variant. is_some ( ) {
251
+ // Downcasts are currently not supported.
252
+ return ;
253
+ } ;
254
+ let new_local = body. local_decls . push ( LocalDecl {
255
+ ty : field_ty,
256
+ user_ty : None ,
257
+ ..decl. clone ( )
258
+ } ) ;
259
+ fragments
260
+ . get_or_insert_with ( local, IndexVec :: new)
261
+ . insert ( field, LocalMode :: Field ( field_ty, new_local) ) ;
262
+ } ) ;
263
+ }
212
264
}
213
265
ReplacementMap { fragments }
214
266
}
@@ -284,14 +336,16 @@ impl<'tcx> ReplacementVisitor<'tcx, '_> {
284
336
let ty = place. ty ( self . local_decls , self . tcx ) . ty ;
285
337
286
338
parts
287
- . map ( |( field, field_ty , replacement_local) | {
339
+ . map ( |( field, replacement_local) | {
288
340
let mut var_debug_info = var_debug_info. clone ( ) ;
289
341
let composite = var_debug_info. composite . get_or_insert_with ( || {
290
342
Box :: new ( VarDebugInfoFragment { ty, projection : Vec :: new ( ) } )
291
343
} ) ;
292
- composite. projection . push ( PlaceElem :: Field ( field, field_ty) ) ;
344
+ let elem = replacement_local. elem ( field) ;
345
+ composite. projection . push ( elem) ;
293
346
294
- var_debug_info. value = VarDebugInfoContents :: Place ( replacement_local. into ( ) ) ;
347
+ let local = replacement_local. local ( ) ;
348
+ var_debug_info. value = VarDebugInfoContents :: Place ( local. into ( ) ) ;
295
349
var_debug_info
296
350
} )
297
351
. collect ( )
@@ -318,7 +372,8 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
318
372
// Duplicate storage and deinit statements, as they pretty much apply to all fields.
319
373
StatementKind :: StorageLive ( l) => {
320
374
if let Some ( final_locals) = self . replacements . place_fragments ( l. into ( ) ) {
321
- for ( _, _, fl) in final_locals {
375
+ for ( _, fl) in final_locals {
376
+ let fl = fl. local ( ) ;
322
377
self . patch . add_statement ( location, StatementKind :: StorageLive ( fl) ) ;
323
378
}
324
379
statement. make_nop ( ) ;
@@ -327,7 +382,8 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
327
382
}
328
383
StatementKind :: StorageDead ( l) => {
329
384
if let Some ( final_locals) = self . replacements . place_fragments ( l. into ( ) ) {
330
- for ( _, _, fl) in final_locals {
385
+ for ( _, fl) in final_locals {
386
+ let fl = fl. local ( ) ;
331
387
self . patch . add_statement ( location, StatementKind :: StorageDead ( fl) ) ;
332
388
}
333
389
statement. make_nop ( ) ;
@@ -336,7 +392,8 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
336
392
}
337
393
StatementKind :: Deinit ( box place) => {
338
394
if let Some ( final_locals) = self . replacements . place_fragments ( place) {
339
- for ( _, _, fl) in final_locals {
395
+ for ( _, fl) in final_locals {
396
+ let fl = fl. local ( ) ;
340
397
self . patch
341
398
. add_statement ( location, StatementKind :: Deinit ( Box :: new ( fl. into ( ) ) ) ) ;
342
399
}
@@ -345,6 +402,35 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
345
402
}
346
403
}
347
404
405
+ // We have `a = [x; N]`
406
+ // We replace it by
407
+ // ```
408
+ // a_0 = x
409
+ // a_1 = x
410
+ // ...
411
+ // ```
412
+ StatementKind :: Assign ( box ( place, Rvalue :: Repeat ( ref mut operand, _) ) ) => {
413
+ if let Some ( local) = place. as_local ( )
414
+ && let Some ( final_locals) = & self . replacements . fragments [ local]
415
+ {
416
+ // Replace mentions of SROA'd locals that appear in the operand.
417
+ self . visit_operand ( & mut * operand, location) ;
418
+
419
+ for & new_local in final_locals. iter ( ) {
420
+ if let Some ( new_local) = new_local {
421
+ let new_local = new_local. local ( ) ;
422
+ let rvalue = Rvalue :: Use ( operand. to_copy ( ) ) ;
423
+ self . patch . add_statement (
424
+ location,
425
+ StatementKind :: Assign ( Box :: new ( ( new_local. into ( ) , rvalue) ) ) ,
426
+ ) ;
427
+ }
428
+ }
429
+ statement. make_nop ( ) ;
430
+ return ;
431
+ }
432
+ }
433
+
348
434
// We have `a = Struct { 0: x, 1: y, .. }`.
349
435
// We replace it by
350
436
// ```
@@ -358,8 +444,10 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
358
444
{
359
445
// This is ok as we delete the statement later.
360
446
let operands = std:: mem:: take ( operands) ;
361
- for ( & opt_ty_local, mut operand) in final_locals. iter ( ) . zip ( operands) {
362
- if let Some ( ( _, new_local) ) = opt_ty_local {
447
+ for ( & new_local, mut operand) in final_locals. iter ( ) . zip ( operands) {
448
+ if let Some ( new_local) = new_local {
449
+ let new_local = new_local. local ( ) ;
450
+
363
451
// Replace mentions of SROA'd locals that appear in the operand.
364
452
self . visit_operand ( & mut operand, location) ;
365
453
@@ -387,8 +475,10 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
387
475
if let Some ( final_locals) = self . replacements . place_fragments ( place) {
388
476
// Put the deaggregated statements *after* the original one.
389
477
let location = location. successor_within_block ( ) ;
390
- for ( field, ty, new_local) in final_locals {
391
- let rplace = self . tcx . mk_place_field ( place, field, ty) ;
478
+ for ( field, new_local) in final_locals {
479
+ let elem = new_local. elem ( field) ;
480
+ let new_local = new_local. local ( ) ;
481
+ let rplace = self . tcx . mk_place_elem ( place, elem) ;
392
482
let rvalue = Rvalue :: Use ( Operand :: Move ( rplace) ) ;
393
483
self . patch . add_statement (
394
484
location,
@@ -414,8 +504,10 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
414
504
Operand :: Constant ( _) => bug ! ( ) ,
415
505
} ;
416
506
if let Some ( final_locals) = self . replacements . place_fragments ( lhs) {
417
- for ( field, ty, new_local) in final_locals {
418
- let rplace = self . tcx . mk_place_field ( rplace, field, ty) ;
507
+ for ( field, new_local) in final_locals {
508
+ let elem = new_local. elem ( field) ;
509
+ let new_local = new_local. local ( ) ;
510
+ let rplace = self . tcx . mk_place_elem ( rplace, elem) ;
419
511
debug ! ( ?rplace) ;
420
512
let rplace = self
421
513
. replacements
0 commit comments