Skip to content

Commit 668e698

Browse files
borsbors
committed
Auto merge of #41418 - hirschenberger:prefetch-intrinsic, r=nagisa
Adding support for the llvm `prefetch` intrinsic Optimize `slice::binary_search` by using prefetching.
2 parents 4ed2eda + f83901b commit 668e698

File tree

5 files changed

+143
-1
lines changed

5 files changed

+143
-1
lines changed

‎src/libcore/intrinsics.rs

+47
Original file line numberDiff line numberDiff line change
@@ -564,8 +564,55 @@ extern "rust-intrinsic" {
564564
pub fn atomic_umax_rel<T>(dst: *mut T, src: T) -> T;
565565
pub fn atomic_umax_acqrel<T>(dst: *mut T, src: T) -> T;
566566
pub fn atomic_umax_relaxed<T>(dst: *mut T, src: T) -> T;
567+
568+
/// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
569+
/// if supported; otherwise, it is a noop.
570+
/// Prefetches have no effect on the behavior of the program but can change its performance
571+
/// characteristics.
572+
///
573+
/// The `locality` argument must be a constant integer and is a temporal locality specifier
574+
/// ranging from (0) - no locality, to (3) - extremely local keep in cache
575+
#[cfg(not(stage0))]
576+
pub fn prefetch_read_data<T>(data: *const T, locality: i32);
577+
/// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
578+
/// if supported; otherwise, it is a noop.
579+
/// Prefetches have no effect on the behavior of the program but can change its performance
580+
/// characteristics.
581+
///
582+
/// The `locality` argument must be a constant integer and is a temporal locality specifier
583+
/// ranging from (0) - no locality, to (3) - extremely local keep in cache
584+
#[cfg(not(stage0))]
585+
pub fn prefetch_write_data<T>(data: *const T, locality: i32);
586+
/// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
587+
/// if supported; otherwise, it is a noop.
588+
/// Prefetches have no effect on the behavior of the program but can change its performance
589+
/// characteristics.
590+
///
591+
/// The `locality` argument must be a constant integer and is a temporal locality specifier
592+
/// ranging from (0) - no locality, to (3) - extremely local keep in cache
593+
#[cfg(not(stage0))]
594+
pub fn prefetch_read_instruction<T>(data: *const T, locality: i32);
595+
/// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
596+
/// if supported; otherwise, it is a noop.
597+
/// Prefetches have no effect on the behavior of the program but can change its performance
598+
/// characteristics.
599+
///
600+
/// The `locality` argument must be a constant integer and is a temporal locality specifier
601+
/// ranging from (0) - no locality, to (3) - extremely local keep in cache
602+
#[cfg(not(stage0))]
603+
pub fn prefetch_write_instruction<T>(data: *const T, locality: i32);
567604
}
568605

606+
// Empty bootstrap implementations for stage0 compilation
607+
#[cfg(stage0)]
608+
pub fn prefetch_read_data<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
609+
#[cfg(stage0)]
610+
pub fn prefetch_write_data<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
611+
#[cfg(stage0)]
612+
pub fn prefetch_read_instruction<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
613+
#[cfg(stage0)]
614+
pub fn prefetch_write_instruction<T>(_data: *const T, _locality: i32) { /* EMPTY */ }
615+
569616
extern "rust-intrinsic" {
570617

571618
pub fn atomic_fence();

‎src/librustc_trans/context.rs

+1
Original file line numberDiff line numberDiff line change
@@ -942,6 +942,7 @@ fn declare_intrinsic(ccx: &CrateContext, key: &str) -> Option<ValueRef> {
942942
ifn!("llvm.x86.seh.recoverfp", fn(i8p, i8p) -> i8p);
943943

944944
ifn!("llvm.assume", fn(i1) -> void);
945+
ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void);
945946

946947
if ccx.sess().opts.debuginfo != NoDebugInfo {
947948
ifn!("llvm.dbg.declare", fn(Type::metadata(ccx), Type::metadata(ccx)) -> void);

‎src/librustc_trans/intrinsic.rs

+12-1
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,18 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bcx: &Builder<'a, 'tcx>,
255255
}
256256
C_nil(ccx)
257257
},
258-
258+
"prefetch_read_data" | "prefetch_write_data" |
259+
"prefetch_read_instruction" | "prefetch_write_instruction" => {
260+
let expect = ccx.get_intrinsic(&("llvm.prefetch"));
261+
let (rw, cache_type) = match name {
262+
"prefetch_read_data" => (0, 1),
263+
"prefetch_write_data" => (1, 1),
264+
"prefetch_read_instruction" => (0, 0),
265+
"prefetch_write_instruction" => (1, 0),
266+
_ => bug!()
267+
};
268+
bcx.call(expect, &[llargs[0], C_i32(ccx, rw), llargs[1], C_i32(ccx, cache_type)], None)
269+
},
259270
"ctlz" | "cttz" | "ctpop" | "bswap" |
260271
"add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" |
261272
"overflowing_add" | "overflowing_sub" | "overflowing_mul" |

‎src/librustc_typeck/check/intrinsic.rs

+8
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,14 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
133133
],
134134
tcx.mk_nil())
135135
}
136+
"prefetch_read_data" | "prefetch_write_data" |
137+
"prefetch_read_instruction" | "prefetch_write_instruction" => {
138+
(1, vec![tcx.mk_ptr(ty::TypeAndMut {
139+
ty: param(0),
140+
mutbl: hir::MutImmutable
141+
}), tcx.types.i32],
142+
tcx.mk_nil())
143+
}
136144
"drop_in_place" => {
137145
(1, vec![tcx.mk_mut_ptr(param(0))], tcx.mk_nil())
138146
}

‎src/test/codegen/prefetch.rs

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// compile-flags: -C no-prepopulate-passes
12+
13+
#![crate_type = "lib"]
14+
#![feature(core_intrinsics)]
15+
16+
use std::intrinsics::{prefetch_read_data, prefetch_write_data,
17+
prefetch_read_instruction, prefetch_write_instruction};
18+
19+
#[no_mangle]
20+
pub fn check_prefetch_read_data(data: &[i8]) {
21+
unsafe {
22+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 0, i32 1)
23+
prefetch_read_data(data.as_ptr(), 0);
24+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 1, i32 1)
25+
prefetch_read_data(data.as_ptr(), 1);
26+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 2, i32 1)
27+
prefetch_read_data(data.as_ptr(), 2);
28+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 3, i32 1)
29+
prefetch_read_data(data.as_ptr(), 3);
30+
}
31+
}
32+
33+
#[no_mangle]
34+
pub fn check_prefetch_write_data(data: &[i8]) {
35+
unsafe {
36+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 0, i32 1)
37+
prefetch_write_data(data.as_ptr(), 0);
38+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 1, i32 1)
39+
prefetch_write_data(data.as_ptr(), 1);
40+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 2, i32 1)
41+
prefetch_write_data(data.as_ptr(), 2);
42+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 3, i32 1)
43+
prefetch_write_data(data.as_ptr(), 3);
44+
}
45+
}
46+
47+
#[no_mangle]
48+
pub fn check_prefetch_read_instruction(data: &[i8]) {
49+
unsafe {
50+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 0, i32 0)
51+
prefetch_read_instruction(data.as_ptr(), 0);
52+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 1, i32 0)
53+
prefetch_read_instruction(data.as_ptr(), 1);
54+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 2, i32 0)
55+
prefetch_read_instruction(data.as_ptr(), 2);
56+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 0, i32 3, i32 0)
57+
prefetch_read_instruction(data.as_ptr(), 3);
58+
}
59+
}
60+
61+
#[no_mangle]
62+
pub fn check_prefetch_write_instruction(data: &[i8]) {
63+
unsafe {
64+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 0, i32 0)
65+
prefetch_write_instruction(data.as_ptr(), 0);
66+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 1, i32 0)
67+
prefetch_write_instruction(data.as_ptr(), 1);
68+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 2, i32 0)
69+
prefetch_write_instruction(data.as_ptr(), 2);
70+
// CHECK: call void @llvm.prefetch(i8* %{{.*}}, i32 1, i32 3, i32 0)
71+
prefetch_write_instruction(data.as_ptr(), 3);
72+
}
73+
}
74+
75+

0 commit comments

Comments
 (0)