Upgrade to MozillaBuild 4.0

.github/workflows/rust.yml

@@ -67,12 +67,17 @@ jobs:
     - uses: actions/checkout@v2
     - name: Install deps
       run: |
-        Start-BitsTransfer -Source https://ftp.mozilla.org/pub/mozilla/libraries/win32/MozillaBuildSetup-3.4.exe -Destination ./MozillaBuildSetup.exe
+        Start-BitsTransfer -Source https://ftp.mozilla.org/pub/mozilla/libraries/win32/MozillaBuildSetup-4.0.2.exe -Destination ./MozillaBuildSetup.exe
         .\MozillaBuildSetup.exe /S | Out-Null
         iwr -useb get.scoop.sh -outfile 'install.ps1'
         .\install.ps1 -RunAsAdmin
         scoop install [email protected] --global
         echo "C:\ProgramData\scoop\shims;C:\Users\runneradmin\scoop\shims" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+    # TODO: Remove this step and start-shell.bat when the MozillaBuild is updated
+    - name: Replace MozillaBuild shell
+      run: |
+        Remove-Item -Path C:\mozilla-build\start-shell.bat
+        Move-Item -Path start-shell.bat -Destination C:\mozilla-build\
     - uses: dtolnay/rust-toolchain@stable
       if: (!contains(matrix.target, 'uwp'))
     - uses: dtolnay/rust-toolchain@master
@@ -82,31 +87,25 @@ jobs:
         components: rust-src
     - name: Build uwp
       if: contains(matrix.target, 'uwp')
-      shell: cmd
       env:
-        MOZTOOLS_PATH: 'C:\mozilla-build\msys\bin;C:\mozilla-build\bin'
-        AUTOCONF: "C:/mozilla-build/msys/local/bin/autoconf-2.13"
+        MOZILLABUILD: 'C:\mozilla-build'
         LINKER: "lld-link.exe"
         CC: "clang-cl.exe"
         CXX: "clang-cl.exe"
-        NATIVE_WIN32_PYTHON: "C:\\mozilla-build\\python2\\python.exe"
-        PYTHON3: "C:\\mozilla-build\\python3\\python3.exe"
-        LIBCLANG_PATH: "C:\\ProgramData\\scoop\\apps\\llvm\\current\\lib"
+        PYTHON3: 'C:\mozilla-build\python3\python3.exe'
+        LIBCLANG_PATH: 'C:\ProgramData\scoop\apps\llvm\current\lib'
       run: |
         rustc --version --verbose
         cargo build --verbose ${{ matrix.features }} -Z build-std=std,panic_abort --target ${{ matrix.target }}
     - name: Build Windows
       if: contains(matrix.target, 'uwp') != true
-      shell: cmd
       env:
-        MOZTOOLS_PATH: 'C:\mozilla-build\msys\bin;C:\mozilla-build\bin'
-        AUTOCONF: "C:/mozilla-build/msys/local/bin/autoconf-2.13"
+        MOZILLABUILD: 'C:\mozilla-build'
         LINKER: "lld-link.exe"
         CC: "clang-cl.exe"
         CXX: "clang-cl.exe"
-        NATIVE_WIN32_PYTHON: "C:\\mozilla-build\\python2\\python.exe"
-        PYTHON3: "C:\\mozilla-build\\python3\\python3.exe"
-        LIBCLANG_PATH: "C:\\ProgramData\\scoop\\apps\\llvm\\current\\lib"
+        PYTHON3: 'C:\mozilla-build\python3\python3.exe'
+        LIBCLANG_PATH: 'C:\ProgramData\scoop\apps\llvm\current\lib'
       run: |
         cargo test --verbose ${{ matrix.features }} --lib
   Integrity:

README.md

@@ -29,30 +29,21 @@ export LIBCLANG_PATH=/usr/lib/clang/4.0/lib
 Under Windows:
 
 1. Follow the directions at
-   https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Build_Instructions/Windows_Prerequisites
+   https://firefox-source-docs.mozilla.org/setup/windows_build.html.
+   (Steps 1.1 and 1.2)
 
-2. Open up a shell configured to use Visual Studio. This could be the
-   one included with Visual Studio (e.g. Visual Studio 2017 / X64 Native
-   Tools Command Prompt for VS 2017) or a shell in which you have run
-```
-"c:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
-```
-
-3. Set the `MOZTOOLS_PATH` environment variable to point to the tools from the Mozilla Build Package:
-```
-set MOZTOOLS_PATH=C:\mozilla-build\msys\bin;C:\mozilla-build\bin
-```
-
-4. Download and install Clang for Windows (64 bit) from https://releases.llvm.org/download.html
+2. Download and install Clang for Windows (64 bit) from https://releases.llvm.org/download.html
    and set the `LIBCLANG_PATH` environment variable to its `lib` directory:
 ```
 set LIBCLANG_PATH=C:\Program Files\LLVM\lib
 ```
 
-5. Set environment variables so the build script can find Python 2.7 and Autoconf 2.13:
+3. Set environment variables so the build script can find the build tools.
 ```
-set AUTOCONF=C:\mozilla-build\msys\local\bin\autoconf-2.13
-set NATIVE_WIN32_PYTHON=C:\mozilla-build\python\python2.7.exe
+set CC=clang-cl.exe
+set CXX=clang-cl.exe
+set LINKER=lld-link.exe
+set MOZILLABUILD=C:\mozilla-build
 ```
 
 You can now build and test the crate using cargo:

mozjs/Cargo.toml

@@ -37,3 +37,7 @@ libz-sys = "1.0"
 bindgen = { version = "0.62", default-features = false, features = ["runtime", "which-rustfmt"] }
 cc = "1.0"
 walkdir = "2"
+
+[target.'cfg(windows)'.build-dependencies]
+ruzstd = "0.3.0"
+tar = "0.4.38"

mozjs/build.rs

@@ -4,11 +4,17 @@
 
 extern crate bindgen;
 extern crate cc;
+#[cfg(windows)]
+extern crate ruzstd;
+#[cfg(windows)]
+extern crate tar;
 extern crate walkdir;
 
 use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fs;
+use std::fs::File;
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::str;
@@ -26,8 +32,8 @@ const ENV_VARS: &'static [&'static str] = &[
     "CXXFLAGS",
     "MAKE",
     "MOZ_TOOLS",
+    "MOZILLABUILD",
     "MOZJS_FORCE_RERUN",
-    "MOZTOOLS_PATH",
     "PYTHON",
     "STLPORT_LIBS",
 ];
@@ -77,21 +83,74 @@ fn main() {
     }
 }
 
-fn find_make() -> OsString {
-    if let Some(make) = env::var_os("MAKE") {
-        make
-    } else {
-        match Command::new("gmake").status() {
-            Ok(gmake) => {
-                if gmake.success() {
-                    OsStr::new("gmake").to_os_string()
+fn find_make() -> Option<OsString> {
+    fn check_make(name: &OsStr) -> bool {
+        let mut cmd = Command::new(name);
+        cmd.arg("--version");
+        cmd.status().map(|status| status.success()).unwrap_or(false)
+    }
+
+    env::var_os("MAKE").or_else(|| {
+        let gmake = OsString::from("gmake");
+        let make = OsString::from("make");
+        if check_make(&gmake) {
+            Some(gmake)
+        } else if check_make(&make) {
+            Some(make)
+        } else {
+            #[cfg(windows)]
+            {
+                let mozmake = OsString::from("mozmake");
+                if check_make(&mozmake) {
+                    Some(mozmake)
                 } else {
-                    OsStr::new("make").to_os_string()
+                    None
                 }
             }
-            Err(_) => OsStr::new("make").to_os_string(),
+            #[cfg(unix)]
+            {
+                None
+            }
         }
-    }
+    })
+}
+
+#[cfg(windows)]
+fn install_mozmake(mozbuild_dir: &Path) {
+    let mozbuild_dir = Path::new(mozbuild_dir);
+    let mozmake_tar_zst_path = mozbuild_dir.join("mozmake.tar.zst");
+
+    let mut curl = Command::new("curl");
+    let result = curl.arg("-SL")
+        .arg("https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.cache.level-1.toolchains.v3.win64-mozmake.latest/artifacts/public/build/mozmake.tar.zst")
+        .args(&["-o", &mozmake_tar_zst_path.display().to_string()])
+        .status()
+        .expect("Failed to install mozmake.");
+    assert!(result.success());
+
+    let mozmake_tar_zst =
+        File::open(&mozmake_tar_zst_path).expect("Failed to open mozmake.tar.zst");
+    let mut mozmake_compressed =
+        ruzstd::StreamingDecoder::new(mozmake_tar_zst).expect("Failed to decode mozmake.tar.zst");
+    let mut mozmake_uncompressed = Vec::new();
+
+    mozmake_compressed
+        .read_to_end(&mut mozmake_uncompressed)
+        .expect("Failed to decode mozmake.tar.zst");
+    let mut archive = tar::Archive::new(&*mozmake_uncompressed);
+    archive
+        .unpack(mozbuild_dir.join("bin"))
+        .expect("Failed to unpack mozmake.tar");
+
+    assert!(mozbuild_dir
+        .join("bin")
+        .join("mozmake")
+        .join("mozmake.exe")
+        .exists());
+    fs::remove_file(&mozmake_tar_zst_path).expect("Failed to remove mozmake.tar.zst");
+
+    let mozmake_lock = mozbuild_dir.join("MOZMAKE_LOCK");
+    fs::write(mozmake_lock, "").expect("Failed to write to mozmake lock.");
 }
 
 fn cc_flags() -> Vec<&'static str> {
@@ -132,20 +191,49 @@ fn cc_flags() -> Vec<&'static str> {
 }
 
 fn build_jsapi(build_dir: &Path) {
+    fn cmd_to_string(cmd: &Command) -> String {
+        let mut cmd_string = format!(r#""{}""#, cmd.get_program().to_string_lossy());
+        for arg in cmd.get_args() {
+            cmd_string.push_str(&format!(r#" "{}""#, arg.to_string_lossy()));
+        }
+        cmd_string
+    }
+
     let target = env::var("TARGET").unwrap();
     let mut make = find_make();
 
-    // Put MOZTOOLS_PATH at the beginning of PATH if specified
-    if let Some(moztools) = env::var_os("MOZTOOLS_PATH") {
-        let path = env::var_os("PATH").unwrap();
-        let mut paths = Vec::new();
-        paths.extend(env::split_paths(&moztools));
-        paths.extend(env::split_paths(&path));
-        let new_path = env::join_paths(paths).unwrap();
-        env::set_var("PATH", &new_path);
-        make = OsStr::new("mozmake").to_os_string();
+    let mozbuild = env::var_os("MOZILLABUILD").map(|mozbuild_env| {
+        if mozbuild_env.is_empty() || mozbuild_env.to_ascii_lowercase() == "true" {
+            env::set_var("MOZILLABUILD", r#"C:\mozilla-build\"#);
+            PathBuf::from(r#"C:\mozilla-build\"#)
+        } else {
+            PathBuf::from(&mozbuild_env)
+        }
+    });
+
+    #[cfg(windows)]
+    {
+        if let Some(mozbuild_dir) = &mozbuild {
+            // Add mozmake to PATH
+            let path = env::var_os("PATH").unwrap();
+            let mut paths = Vec::new();
+            paths.push(mozbuild_dir.join("bin").join("mozmake").into());
+            paths.extend(env::split_paths(&path));
+            let new_path = env::join_paths(paths).unwrap();
+            env::set_var("PATH", &new_path);
+
+            if make.is_none() {
+                // Install mozmake if not installed
+                if !mozbuild_dir.join("MOZMAKE_LOCK").exists() {
+                    install_mozmake(&mozbuild_dir);
+                }
+
+                make = Some(OsString::from("mozmake"));
+            }
+        }
     }
 
+    let make = make.expect("Install `make` or `gmake`");
     let mut cmd = Command::new(make);
 
     let encoding_c_mem_include_dir = env::var("DEP_ENCODING_C_MEM_INCLUDE_DIR").unwrap();
@@ -155,10 +243,10 @@ fn build_jsapi(build_dir: &Path) {
     ));
     cppflags.push(" ");
     cppflags.push(env::var_os("CPPFLAGS").unwrap_or_default());
-    cmd.env("CPPFLAGS", cppflags);
+    cmd.env("CPPFLAGS", &cppflags);
 
     if let Some(makeflags) = env::var_os("CARGO_MAKEFLAGS") {
-        cmd.env("MAKEFLAGS", makeflags);
+        cmd.env("MAKEFLAGS", &makeflags);
     }
 
     if target.contains("apple") || target.contains("freebsd") {
@@ -167,15 +255,51 @@ fn build_jsapi(build_dir: &Path) {
 
     let cargo_manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
 
-    let result = cmd
-        .args(&["-R", "-f"])
+    cmd.args(&["-R", "-f"])
         .arg(cargo_manifest_dir.join("makefile.cargo"))
         .current_dir(&build_dir)
         .env("SRC_DIR", &cargo_manifest_dir.join("mozjs"))
-        .env("NO_RUST_PANIC_HOOK", "1")
-        .status()
-        .expect("Failed to run `make`");
-    assert!(result.success());
+        .env("NO_RUST_PANIC_HOOK", "1");
+
+    #[cfg(not(windows))]
+    {
+        let result = cmd.status().expect("Failed to run `make`");
+        assert!(result.success());
+    }
+
+    #[cfg(windows)]
+    {
+        if let Some(mozbuild_dir) = mozbuild {
+            // Create script that runs mozmake
+            let script_path = build_dir.join("mozbuild.sh");
+            let script = cmd_to_string(&cmd);
+            fs::write(&script_path, script).expect("Failed to write to mozbuild.sh");
+
+            let start_shell = mozbuild_dir.join("start-shell.bat");
+            let mut shell = Command::new(start_shell.as_os_str());
+            shell
+                .arg("-here")
+                .arg("-use-full-path")
+                .arg(script_path.display().to_string());
+
+            for (key, value) in cmd.get_envs() {
+                if let Some(value) = value {
+                    shell.env(key, value);
+                }
+            }
+
+            if let Some(current_dir) = cmd.get_current_dir() {
+                shell.current_dir(current_dir);
+            }
+
+            let result = shell.status().expect("Failed to run `make`");
+            assert!(result.success());
+        } else {
+            let result = cmd.status().expect("Failed to run `make`");
+            assert!(result.success());
+        }
+    }
+
 
     println!(
         "cargo:rustc-link-search=native={}/js/src/build",