Bug 1465709 - Hook rust OOM handler on rustc 1.28. r=froydnj

Bug 1458161 added a rust OOM handler based on an unstable API that was
removed in 1.27, replaced with something that didn't allow to get the
failed allocation size.

Latest 1.28 nightly (2018-06-13) has
rust-lang/rust#50880,
rust-lang/rust#51264 and
rust-lang/rust#51241 merged, which allow to
hook the OOM handler and get the failed allocation size again.

Because this is still an unstable API, we explicitly depend on strict
versions of rustc. We also explicitly error out if automation builds
end up using a rustc version that doesn't allow us to get the allocation
size for rust OOM, because we don't want that to happen without knowing.

UltraBlame original commit: 5182bca90d0609f182d5a7b6b48ed2ffbbce32c2

Author: marco-c

toolkit/crashreporter/nsExceptionHandler.cpp

@@ -765,6 +765,11 @@ install_rust_panic_hook
 (
 )
 ;
+void
+install_rust_oom_hook
+(
+)
+;
 bool
 get_rust_panic_reason
 (
@@ -7332,6 +7337,10 @@ install_rust_panic_hook
 (
 )
 ;
+install_rust_oom_hook
+(
+)
+;
 InitThreadAnnotation
 (
 )

toolkit/library/gtest/rust/Cargo.toml

@@ -143,6 +143,17 @@ shared
 oom_with_global_alloc
 "
 ]
+oom_with_hook
+=
+[
+"
+gkrust
+-
+shared
+/
+oom_with_hook
+"
+]
 moz_memory
 =
 [

toolkit/library/rust/Cargo.toml

@@ -141,6 +141,17 @@ shared
 oom_with_global_alloc
 "
 ]
+oom_with_hook
+=
+[
+"
+gkrust
+-
+shared
+/
+oom_with_hook
+"
+]
 moz_memory
 =
 [

toolkit/library/rust/gkrust-features.mozbuild

@@ -310,3 +310,88 @@ gkrust_features
 oom_with_global_alloc
 '
 ]
+elif
+CONFIG
+[
+'
+RUSTC_VERSION
+'
+]
+>
+=
+"
+1
+.
+28
+"
+and
+CONFIG
+[
+'
+RUSTC_VERSION
+'
+]
+<
+"
+1
+.
+29
+"
+:
+gkrust_features
++
+=
+[
+'
+oom_with_hook
+'
+]
+elif
+not
+CONFIG
+[
+'
+MOZ_AUTOMATION
+'
+]
+:
+#
+We
+don
+'
+t
+want
+builds
+on
+automation
+to
+unwillingly
+stop
+annotating
+OOM
+#
+crash
+reports
+from
+rust
+.
+error
+(
+'
+Builds
+on
+automation
+must
+use
+a
+version
+of
+rust
+that
+supports
+OOM
+'
+'
+hooking
+'
+)

toolkit/library/rust/shared/Cargo.toml

@@ -679,6 +679,10 @@ oom_with_global_alloc
 =
 [
 ]
+oom_with_hook
+=
+[
+]
 moz_memory
 =
 [

toolkit/library/rust/shared/build.rs

@@ -7,11 +7,19 @@ main
 [
 cfg
 (
+any
+(
 feature
 =
 "
 oom_with_global_alloc
 "
+feature
+=
+"
+oom_with_hook
+"
+)
 )
 ]
 println

toolkit/library/rust/shared/lib.rs

@@ -18,6 +18,22 @@ allocator_api
 )
 ]
 #
+!
+[
+cfg_attr
+(
+feature
+=
+"
+oom_with_hook
+"
+feature
+(
+oom_hook
+)
+)
+]
+#
 [
 cfg
 (
@@ -1153,3 +1169,114 @@ global_alloc
 :
 GeckoHeap
 ;
+#
+[
+cfg
+(
+feature
+=
+"
+oom_with_hook
+"
+)
+]
+mod
+oom_hook
+{
+use
+std
+:
+:
+alloc
+:
+:
+{
+Layout
+set_oom_hook
+}
+;
+extern
+"
+C
+"
+{
+fn
+GeckoHandleOOM
+(
+size
+:
+usize
+)
+-
+>
+!
+;
+}
+pub
+fn
+hook
+(
+layout
+:
+Layout
+)
+{
+unsafe
+{
+GeckoHandleOOM
+(
+layout
+.
+size
+(
+)
+)
+;
+}
+}
+pub
+fn
+install
+(
+)
+{
+set_oom_hook
+(
+hook
+)
+;
+}
+}
+#
+[
+no_mangle
+]
+pub
+extern
+"
+C
+"
+fn
+install_rust_oom_hook
+(
+)
+{
+#
+[
+cfg
+(
+feature
+=
+"
+oom_with_hook
+"
+)
+]
+oom_hook
+:
+:
+install
+(
+)
+;
+}