Auto merge of rust-lang#50000 - michaelwoerister:cross-lang-lto, r=alexcrichton

Add some groundwork for cross-language LTO.

Implements part of rust-lang#49879:
- Adds a `-Z cross-lang-lto` flag to rustc
- Makes sure that bitcode is embedded in object files if the flag is set.

This should already allow for using cross language LTO for staticlibs (where one has to invoke the linker manually anyway). However, `rustc` will not try to enable LTO for its own linker invocations yet.

r? @alexcrichton

Author: bors

src/bootstrap/test.rs

@@ -835,7 +835,7 @@ test!(RunFailFullDepsPretty {
     host: true
 });
 
-default_test!(RunMake {
+host_test!(RunMake {
     path: "src/test/run-make",
     mode: "run-make",
     suite: "run-make"
@@ -1022,7 +1022,7 @@ impl Step for Compiletest {
 
             // Only pass correct values for these flags for the `run-make` suite as it
             // requires that a C++ compiler was configured which isn't always the case.
-            if !builder.config.dry_run && suite == "run-make-fulldeps" {
+            if !builder.config.dry_run && mode == "run-make" {
                 let llvm_components = output(Command::new(&llvm_config).arg("--components"));
                 let llvm_cxxflags = output(Command::new(&llvm_config).arg("--cxxflags"));
                 cmd.arg("--cc").arg(builder.cc(target))
@@ -1035,13 +1035,13 @@ impl Step for Compiletest {
                 }
             }
         }
-        if suite == "run-make-fulldeps" && !builder.config.llvm_enabled {
+        if mode == "run-make" && !builder.config.llvm_enabled {
             builder.info(
                 &format!("Ignoring run-make test suite as they generally don't work without LLVM"));
             return;
         }
 
-        if suite != "run-make-fulldeps" {
+        if mode != "run-make" {
             cmd.arg("--cc").arg("")
                .arg("--cxx").arg("")
                .arg("--cflags").arg("")

src/bootstrap/tool.rs

@@ -10,6 +10,7 @@
 
 use std::fs;
 use std::env;
+use std::iter;
 use std::path::PathBuf;
 use std::process::{Command, exit};
 
@@ -593,7 +594,7 @@ impl<'a> Builder<'a> {
     /// right location to run `compiler`.
     fn prepare_tool_cmd(&self, compiler: Compiler, cmd: &mut Command) {
         let host = &compiler.host;
-        let mut paths: Vec<PathBuf> = vec![
+        let mut lib_paths: Vec<PathBuf> = vec![
             PathBuf::from(&self.sysroot_libdir(compiler, compiler.host)),
             self.cargo_out(compiler, Mode::Tool, *host).join("deps"),
         ];
@@ -610,11 +611,46 @@ impl<'a> Builder<'a> {
                 }
                 for path in env::split_paths(v) {
                     if !curpaths.contains(&path) {
-                        paths.push(path);
+                        lib_paths.push(path);
                     }
                 }
             }
         }
-        add_lib_path(paths, cmd);
+
+        // Add the llvm/bin directory to PATH since it contains lots of
+        // useful, platform-independent tools
+        if let Some(llvm_bin_path) = self.llvm_bin_path() {
+            if host.contains("windows") {
+                // On Windows, PATH and the dynamic library path are the same,
+                // so we just add the LLVM bin path to lib_path
+                lib_paths.push(llvm_bin_path);
+            } else {
+                let old_path = env::var_os("PATH").unwrap_or_default();
+                let new_path = env::join_paths(iter::once(llvm_bin_path)
+                        .chain(env::split_paths(&old_path)))
+                    .expect("Could not add LLVM bin path to PATH");
+                cmd.env("PATH", new_path);
+            }
+        }
+
+        add_lib_path(lib_paths, cmd);
+    }
+
+    fn llvm_bin_path(&self) -> Option<PathBuf> {
+        if self.config.llvm_enabled && !self.config.dry_run {
+            let llvm_config = self.ensure(native::Llvm {
+                target: self.config.build,
+                emscripten: false,
+            });
+
+            // Add the llvm/bin directory to PATH since it contains lots of
+            // useful, platform-independent tools
+            let llvm_bin_path = llvm_config.parent()
+                .expect("Expected llvm-config to be contained in directory");
+            assert!(llvm_bin_path.is_dir());
+            Some(llvm_bin_path.to_path_buf())
+        } else {
+            None
+        }
     }
 }

src/librustc/session/config.rs

@@ -1295,6 +1295,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
           "make the current crate share its generic instantiations"),
     chalk: bool = (false, parse_bool, [TRACKED],
           "enable the experimental Chalk-based trait solving engine"),
+    cross_lang_lto: bool = (false, parse_bool, [TRACKED],
+          "generate build artifacts that are compatible with linker-based LTO."),
 }
 
 pub fn default_lib_output() -> CrateType {

src/librustc_trans/back/write.rs

@@ -293,7 +293,8 @@ impl ModuleConfig {
         self.inline_threshold = sess.opts.cg.inline_threshold;
         self.obj_is_bitcode = sess.target.target.options.obj_is_bitcode;
         let embed_bitcode = sess.target.target.options.embed_bitcode ||
-            sess.opts.debugging_opts.embed_bitcode;
+                            sess.opts.debugging_opts.embed_bitcode ||
+                            sess.opts.debugging_opts.cross_lang_lto;
         if embed_bitcode {
             match sess.opts.optimize {
                 config::OptLevel::No |
@@ -841,13 +842,18 @@ unsafe fn embed_bitcode(cgcx: &CodegenContext,
         "rustc.embedded.module\0".as_ptr() as *const _,
     );
     llvm::LLVMSetInitializer(llglobal, llconst);
-    let section = if cgcx.opts.target_triple.triple().contains("-ios") {
+
+    let is_apple = cgcx.opts.target_triple.triple().contains("-ios") ||
+                   cgcx.opts.target_triple.triple().contains("-darwin");
+
+    let section = if is_apple {
         "__LLVM,__bitcode\0"
     } else {
         ".llvmbc\0"
     };
     llvm::LLVMSetSection(llglobal, section.as_ptr() as *const _);
     llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage);
+    llvm::LLVMSetGlobalConstant(llglobal, llvm::True);
 
     let llconst = C_bytes_in_context(llcx, &[]);
     let llglobal = llvm::LLVMAddGlobal(
@@ -856,7 +862,7 @@ unsafe fn embed_bitcode(cgcx: &CodegenContext,
         "rustc.embedded.cmdline\0".as_ptr() as *const _,
     );
     llvm::LLVMSetInitializer(llglobal, llconst);
-    let section = if cgcx.opts.target_triple.triple().contains("-ios") {
+    let section = if  is_apple {
         "__LLVM,__cmdline\0"
     } else {
         ".llvmcmd\0"
@@ -1350,6 +1356,10 @@ fn execute_work_item(cgcx: &CodegenContext,
             // settings.
             let needs_lto = needs_lto && mtrans.kind != ModuleKind::Metadata;
 
+            // Don't run LTO passes when cross-lang LTO is enabled. The linker
+            // will do that for us in this case.
+            let needs_lto = needs_lto && !cgcx.opts.debugging_opts.cross_lang_lto;
+
             if needs_lto {
                 Ok(WorkItemResult::NeedsLTO(mtrans))
             } else {

src/test/run-make/cross-lang-lto/Makefile

@@ -0,0 +1,53 @@
+
+# min-llvm-version 4.0
+# ignore-mingw
+
+-include ../../run-make-fulldeps/tools.mk
+
+# This test makes sure that the expected .llvmbc sections for use by
+# linker-based LTO are available in object files when compiling with
+# -Z cross-lang-lto
+
+LLVMBC_SECTION_NAME=\\.llvmbc
+
+ifeq ($(UNAME),Darwin)
+	LLVMBC_SECTION_NAME=__bitcode
+endif
+
+
+OBJDUMP=llvm-objdump
+SECTION_HEADERS=$(OBJDUMP) -section-headers
+
+BUILD_LIB=$(RUSTC) lib.rs -Copt-level=2 -Z cross-lang-lto -Ccodegen-units=1
+
+BUILD_EXE=$(RUSTC) main.rs -Copt-level=2 -Z cross-lang-lto -Ccodegen-units=1 --emit=obj
+
+all: staticlib staticlib-fat-lto staticlib-thin-lto rlib exe cdylib rdylib
+
+staticlib: lib.rs
+	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib.a
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib.a | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+staticlib-fat-lto: lib.rs
+	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib-fat-lto.a -Clto=fat
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib-fat-lto.a | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+staticlib-thin-lto: lib.rs
+	$(BUILD_LIB) --crate-type=staticlib -o $(TMPDIR)/liblib-thin-lto.a -Clto=thin
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib-thin-lto.a | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+rlib: lib.rs
+	$(BUILD_LIB) --crate-type=rlib -o $(TMPDIR)/liblib.rlib
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/liblib.rlib | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+cdylib: lib.rs
+	$(BUILD_LIB) --crate-type=cdylib --emit=obj -o $(TMPDIR)/cdylib.o
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/cdylib.o | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+rdylib: lib.rs
+	$(BUILD_LIB) --crate-type=dylib --emit=obj -o $(TMPDIR)/rdylib.o
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/rdylib.o | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]
+
+exe: lib.rs
+	$(BUILD_EXE) -o $(TMPDIR)/exe.o
+	[ "$$($(SECTION_HEADERS) $(TMPDIR)/exe.o | grep -c $(LLVMBC_SECTION_NAME))" -ne "0" ]

src/test/run-make/cross-lang-lto/lib.rs

@@ -0,0 +1,14 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[no_mangle]
+pub extern "C" fn foo() {
+    println!("abc");
+}

src/test/run-make/cross-lang-lto/main.rs

@@ -0,0 +1,13 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+fn main() {
+    println!("Hello World");
+}

src/tools/compiletest/src/header.rs

@@ -425,6 +425,15 @@ fn iter_header(testfile: &Path, cfg: Option<&str>, it: &mut FnMut(&str)) {
     if testfile.is_dir() {
         return;
     }
+
+    let comment = if testfile.to_string_lossy().ends_with(".rs") {
+        "//"
+    } else {
+        "#"
+    };
+
+    let comment_with_brace = comment.to_string() + "[";
+
     let rdr = BufReader::new(File::open(testfile).unwrap());
     for ln in rdr.lines() {
         // Assume that any directives will be found before the first
@@ -434,10 +443,11 @@ fn iter_header(testfile: &Path, cfg: Option<&str>, it: &mut FnMut(&str)) {
         let ln = ln.trim();
         if ln.starts_with("fn") || ln.starts_with("mod") {
             return;
-        } else if ln.starts_with("//[") {
+        } else if ln.starts_with(&comment_with_brace) {
             // A comment like `//[foo]` is specific to revision `foo`
             if let Some(close_brace) = ln.find(']') {
-                let lncfg = &ln[3..close_brace];
+                let open_brace = ln.find('[').unwrap();
+                let lncfg = &ln[open_brace + 1 .. close_brace];
                 let matches = match cfg {
                     Some(s) => s == &lncfg[..],
                     None => false,
@@ -446,11 +456,11 @@ fn iter_header(testfile: &Path, cfg: Option<&str>, it: &mut FnMut(&str)) {
                     it(ln[(close_brace + 1) ..].trim_left());
                 }
             } else {
-                panic!("malformed condition directive: expected `//[foo]`, found `{}`",
-                       ln)
+                panic!("malformed condition directive: expected `{}foo]`, found `{}`",
+                        comment_with_brace, ln)
             }
-        } else if ln.starts_with("//") {
-            it(ln[2..].trim_left());
+        } else if ln.starts_with(comment) {
+            it(ln[comment.len() ..].trim_left());
         }
     }
     return;

src/tools/compiletest/src/main.rs

@@ -614,7 +614,12 @@ pub fn is_test(file_name: &OsString) -> bool {
 }
 
 pub fn make_test(config: &Config, testpaths: &TestPaths) -> test::TestDescAndFn {
-    let early_props = EarlyProps::from_file(config, &testpaths.file);
+
+    let early_props = if config.mode == Mode::RunMake {
+        EarlyProps::from_file(config, &testpaths.file.join("Makefile"))
+    } else {
+        EarlyProps::from_file(config, &testpaths.file)
+    };
 
     // The `should-fail` annotation doesn't apply to pretty tests,
     // since we run the pretty printer across all tests by default.