diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d00c9a65..c40de4ccc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,15 +264,15 @@ jobs: fail-fast: false matrix: job: - - { os: macos-12 } - release: ["13.2"] + - { os: ubuntu-22.04 } steps: - uses: actions/checkout@v4 - name: Prepare, build and test - uses: vmactions/freebsd-vm@v0 + uses: vmactions/freebsd-vm@v1.0.2 with: mem: 8192 usesh: true + sync: rsync copyback: false prepare: pkg install -y ca_root_nss curl gmake gtar pot sudo run: | @@ -289,7 +289,7 @@ jobs: TEST_USER=tester TEST_USER_HOME="/opt/$TEST_USER" REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/Users/runner/work/${REPO_NAME}" + WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" export WORKSPACE # diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..f90466bed --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: +- repo: local + hooks: + - id: rust-linting + name: Rust linting + description: Run cargo fmt on files included in the commit. + entry: cargo +nightly fmt -- + pass_filenames: true + types: [file, rust] + language: system + - id: rust-clippy + name: Rust clippy + description: Run cargo clippy on files included in the commit. + entry: cargo +nightly clippy --workspace --all-targets --all-features -- + pass_filenames: false + types: [file, rust] + language: system diff --git a/Cargo.lock b/Cargo.lock index f300fc1a7..148fc3b3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] @@ -41,12 +41,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstyle" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" - [[package]] name = "anstyle" version = "1.0.0" @@ -88,9 +82,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "ascii" @@ -100,11 +94,11 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "assert_cmd" -version = "2.0.10" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b2340f55d9661d76793b2bfc2eb0e62689bd79d067a95707ea762afd5e9dd" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" dependencies = [ - "anstyle 0.3.5", + "anstyle", "bstr", "doc-comment", "predicates", @@ -178,9 +172,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.3" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64ct" @@ -224,16 +218,15 @@ checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" [[package]] name = "blake3" -version = "1.3.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", - "arrayvec 0.7.2", + "arrayvec 0.7.4", "cc", "cfg-if 1.0.0", "constant_time_eq", - "digest", ] [[package]] @@ -275,15 +268,15 @@ checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cc" @@ -409,9 +402,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" @@ -548,22 +541,23 @@ dependencies = [ [[package]] name = "directories" -version = "5.0.0" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74be3be809c18e089de43bdc504652bb2bc473fca8756131f8689db8cf079ba9" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04414300db88f70d74c5ff54e50f9e1d1737d9a5b90f53fcf2e95ca2a9ab554b" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", + "option-ext", "redox_users", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -743,13 +737,13 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "windows-sys 0.48.0", ] @@ -761,9 +755,9 @@ checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide 0.7.1", @@ -997,15 +991,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.1" @@ -1089,9 +1074,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1235,7 +1220,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi", "io-lifetimes", "rustix 0.37.7", "windows-sys 0.48.0", @@ -1258,9 +1243,9 @@ checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -1514,16 +1499,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", "libc", "memoffset", "pin-utils", - "static_assertions", ] [[package]] @@ -1593,11 +1577,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] @@ -1628,9 +1612,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opendal" @@ -1642,7 +1626,7 @@ dependencies = [ "async-compat", "async-trait", "backon", - "base64 0.21.3", + "base64 0.21.5", "bb8", "bytes", "chrono", @@ -1721,6 +1705,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-multimap" version = "0.7.1" @@ -1755,7 +1745,7 @@ checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "windows-sys 0.42.0", ] @@ -1855,7 +1845,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" dependencies = [ - "anstyle 1.0.0", + "anstyle", "difflib", "float-cmp", "itertools", @@ -2023,6 +2013,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -2030,15 +2029,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.7.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", @@ -2053,9 +2052,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "reqsign" @@ -2065,7 +2064,7 @@ checksum = "1ad14258ddd8ef6e564d57a94613e138cc9c21ef8a1fec547206d853213c7959" dependencies = [ "anyhow", "async-trait", - "base64 0.21.3", + "base64 0.21.5", "chrono", "form_urlencoded", "hex", @@ -2094,7 +2093,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.3", + "base64 0.21.5", "bytes", "encoding_rs", "futures-core", @@ -2170,21 +2169,20 @@ dependencies = [ [[package]] name = "rouille" -version = "3.6.1" +version = "3.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f86e4c51a773f953f02bbab5fd049f004bfd384341d62da2a079aff812ab176" +checksum = "3716fbf57fc1084d7a706adf4e445298d123e4a44294c4e8213caf1b85fcc921" dependencies = [ "base64 0.13.1", "chrono", "filetime", "multipart", - "num_cpus", "percent-encoding", "rand", "serde", "serde_derive", "serde_json", - "sha1", + "sha1_smol", "threadpool", "time", "tiny_http", @@ -2335,13 +2333,13 @@ dependencies = [ [[package]] name = "sccache" -version = "0.7.0" +version = "0.7.4" dependencies = [ "anyhow", "ar", "assert_cmd", "async-trait", - "base64 0.21.3", + "base64 0.21.5", "bincode", "blake3", "byteorder", @@ -2371,7 +2369,7 @@ dependencies = [ "memchr", "memmap2", "mime", - "nix 0.26.2", + "nix 0.26.4", "num_cpus", "number_prefix", "object", @@ -2395,6 +2393,7 @@ dependencies = [ "tar", "temp-env", "tempfile", + "test-case", "thirtyfour_sync", "tokio", "tokio-serde", @@ -2463,24 +2462,24 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.159" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", @@ -2512,9 +2511,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" dependencies = [ "serde", ] @@ -2665,12 +2664,6 @@ dependencies = [ "der", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "stringmatch" version = "0.3.3" @@ -2725,9 +2718,9 @@ dependencies = [ [[package]] name = "syslog" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978044cc68150ad5e40083c9f6a725e6fd02d7ba1bcf691ec2ff0d66c0b41acc" +checksum = "7434e95bcccce1215d30f4bf84fe8c00e8de1b9be4fb736d747ca53d36e7f96f" dependencies = [ "error-chain", "hostname", @@ -2764,7 +2757,7 @@ checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if 1.0.0", "fastrand", - "redox_syscall", + "redox_syscall 0.2.16", "rustix 0.36.16", "windows-sys 0.42.0", ] @@ -2794,6 +2787,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" +[[package]] +name = "test-case" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8f1e820b7f1d95a0cdbf97a5df9de10e1be731983ab943e56703ac1b8e9d425" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c25e2cb8f5fcd7318157634e8838aa6f7e4715c96637f969fabaccd1ef5462" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "test-case-macros" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cfd7bbc88a0104e304229fba519bdc45501a30b760fb72240342f1289ad257" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", + "test-case-core", +] + [[package]] name = "thirtyfour" version = "0.27.3" @@ -3027,9 +3055,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.6" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +checksum = "8ff9e3abce27ee2c9a37f9ad37238c1bdd4e789c84ba37df76aa4d528f5072cc" dependencies = [ "serde", "serde_spanned", @@ -3039,18 +3067,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ "indexmap 2.0.0", "serde", @@ -3318,9 +3346,9 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -3734,20 +3762,19 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 6c1d8ebde..546205bb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" name = "sccache" rust-version = "1.67.1" -version = "0.7.0" +version = "0.7.4" categories = ["command-line-utilities", "development-tools::build-utils"] description = "Sccache is a ccache-like tool. It is used as a compiler wrapper and avoids compilation when possible, storing a cache in a remote storage using various cloud storage." @@ -30,11 +30,11 @@ async-trait = "0.1" base64 = "0.21" bincode = "1" blake3 = "1" -byteorder = "1.0" +byteorder = "1.5" bytes = "1" chrono = "0.4" clap = { version = "4.1.11", features = ["derive", "env", "wrap_help"] } -directories = "5.0.0" +directories = "5.0.1" encoding = "0.2" env_logger = "0.10" filetime = "0.2" @@ -47,21 +47,21 @@ gzp = { version = "0.11.3", default-features = false, features = [ "deflate_rust", ] } http = "0.2" -hyper = { version = "0.14.25", optional = true, features = ["server"] } +hyper = { version = "0.14.27", optional = true, features = ["server"] } is-terminal = "0.4.7" jobserver = "0.1" jwt = { package = "jsonwebtoken", version = "8", optional = true } -libc = "0.2.140" +libc = "0.2.149" linked-hash-map = "0.5" log = "0.4" memchr = "2" -num_cpus = "1.15" +num_cpus = "1.16" number_prefix = "0.4" -once_cell = "1.17" +once_cell = "1.18" opendal = { version = "0.40.0", optional = true } openssl = { version = "0.10.55", optional = true } rand = "0.8.4" -regex = "1.7.3" +regex = "1.8.4" reqsign = { version = "0.14.3", optional = true } reqwest = { version = "0.11", features = [ "json", @@ -90,7 +90,7 @@ tokio = { version = "1", features = [ ] } tokio-serde = "0.8" tokio-util = { version = "0.7", features = ["codec", "io"] } -toml = "0.7" +toml = "0.8" tower = "0.4" trust-dns-resolver = { version = "0.22.0", features = [ "dnssec-ring", @@ -103,26 +103,28 @@ walkdir = "2" # by default which pulls in an outdated failure version which = { version = "4", default-features = false } zip = { version = "0.6", default-features = false } -zstd = "0.12" +zstd = "0.13" # dist-server only memmap2 = "0.9.0" -nix = { version = "0.26.2", optional = true } +nix = { version = "0.26.4", optional = true } object = "0.30" -rouille = { version = "3.5", optional = true, default-features = false, features = [ +rouille = { version = "3.6", optional = true, default-features = false, features = [ "ssl", ] } syslog = { version = "6", optional = true } version-compare = { version = "0.1.1", optional = true } [dev-dependencies] -assert_cmd = "2.0.10" +assert_cmd = "2.0.12" cc = "1.0" chrono = "0.4.26" +filetime = "0.2" itertools = "0.10" predicates = "=3.0.3" serial_test = "2.0" temp-env = "0.3.4" +test-case = "3.2.1" thirtyfour_sync = "0.27" [target.'cfg(unix)'.dependencies] diff --git a/README.md b/README.md index 6e5bd6a1d..cf5cd660f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ sccache includes support for caching the compilation of C/C++ code, [Rust](docs/ sccache also provides [icecream](https://github.com/icecc/icecream)-style distributed compilation (automatic packaging of local toolchains) for all supported compilers (including Rust). The distributed compilation system includes several security features that icecream lacks such as authentication, transport layer encryption, and sandboxed compiler execution on build servers. See [the distributed quickstart](docs/DistributedQuickstart.md) guide for more information. -sccache is also available as a [GitHub Actions](https://github.com/marketplace/actions/sccache-action) to faciliate the deployment using GitHub Actions cache. +sccache is also available as a [GitHub Actions](https://github.com/marketplace/actions/sccache-action) to facilitate the deployment using GitHub Actions cache. --- diff --git a/docs/Configuration.md b/docs/Configuration.md index acfc410ed..dcc0a85bc 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -99,7 +99,7 @@ configuration variables * `SCCACHE_DIR` local on disk artifact cache directory * `SCCACHE_CACHE_SIZE` maximum size of the local on disk cache i.e. `2G` - default is 10G -* `SCCACHE_PREPROCESSOR_MODE` enable/disable preprocessor caching (see [the local doc](Local.md)) +* `SCCACHE_DIRECT` enable/disable preprocessor caching (see [the local doc](Local.md)) #### s3 compatible diff --git a/docs/Local.md b/docs/Local.md index d8fb381a6..89525fbf0 100644 --- a/docs/Local.md +++ b/docs/Local.md @@ -10,6 +10,11 @@ The local storage only supports a single sccache server at a time. Multiple conc This is inspired by [ccache's direct mode](https://ccache.dev/manual/3.7.9.html#_the_direct_mode) and works roughly the same. +It can be set with by setting the variable: +``` +SCCACHE_DIRECT=true +``` + In preprocessor cache mode, sccache caches the preprocessor step for C/C++ whenever possible. This can make the compilation a lot faster, since the preprocessor accounts for a non-negligible amount of time in the entire compile chain. In order to cache the preprocessor step sccache needs to remember, among other things, all files included by the given input file. To quote ccache's documentation: @@ -25,7 +30,7 @@ Preprocessor cache mode will be disabled if any of the following holds: Configuration options and their default values: -- `use_preprocessor_cache_mode`: `false`. Whether to use preprocessor cache mode entirely. +- `use_preprocessor_cache_mode`: `true`. Whether to use preprocessor cache mode entirely. - `file_stat_matches`: `false`. If false, only compare header files by hashing their contents. If true, will use size + ctime + mtime to check whether a file has changed. See other flags below for more control over this behavior. - `use_ctime_for_stat`: `true`. If true, uses the ctime (file status change on UNIX, creation time on Windows) to check that a file has/hasn't changed. Can be useful to disable when backdating modification times in a controlled manner. diff --git a/docs/ResponseFiles.md b/docs/ResponseFiles.md index 9c0ccd027..bb20a62bf 100644 --- a/docs/ResponseFiles.md +++ b/docs/ResponseFiles.md @@ -27,10 +27,10 @@ Per the [MSVC docs](https://learn.microsoft.com/en-us/cpp/build/reference/cl-com 4. The `/link` directive has special treatment: 1. Entering an @file: if the `/link` option is provided prior to an `@file` in the command line, the `/link` directive does not affect any options within the `@file`. 2. Newlines: A `/link` directive provided in an `@file` on one line does not affect the next line. - 3. Exitting an @file: A `/link` directive on the final line of a response file does not affect options following the `@file` option in the command line. + 3. Exiting an @file: A `/link` directive on the final line of a response file does not affect options following the `@file` option in the command line. 5. A response file cannot contain additional `@file` options, they are not recursive. (found in a [separate doc](https://learn.microsoft.com/en-us/cpp/build/reference/at-specify-a-compiler-response-file?view=msvc-170)) 6. (implied) options can be wrapped in double-quotes (`"`), which allows whitespace to be preserved within the option -The msvc implementaion in sccache supports all of these **except** #4, because sccache doesn't accept the `/link` directive. +The msvc implementation in sccache supports all of these **except** #4, because sccache doesn't accept the `/link` directive. Additionally, because `msbuild` generates response files using an encoding other than `utf-8`, all text files under the [WHATWG encoding standard](https://encoding.spec.whatwg.org/) are supported. This includes both `utf-8` and `utf-16`. diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 796e2a47e..881738fef 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -18,7 +18,7 @@ description: | sandboxed compiler execution on build servers. See the distributed quickstart guide for more information. - sccache is also available as a GitHub Actions to faciliate the deployment + sccache is also available as a GitHub Actions to facilitate the deployment using GitHub Actions cache. website: https://github.com/mozilla/sccache contact: https://github.com/mozilla/sccache/issues diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 717cde09a..c31761223 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -366,7 +366,7 @@ pub trait Storage: Send + Sync { /// Return the config for preprocessor cache mode if applicable fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { - // Disabled by default, only enabled in local mode + // Enable by default, only in local mode PreprocessorCacheModeConfig::default() } /// Return the preprocessor cache entry for a given preprocessor key, @@ -431,6 +431,16 @@ impl Default for PreprocessorCacheModeConfig { } } +impl PreprocessorCacheModeConfig { + /// Return a default [`Self`], but with the cache active. + pub fn activated() -> Self { + Self { + use_preprocessor_cache_mode: true, + ..Default::default() + } + } +} + /// Implement storage for operator. #[cfg(any(feature = "s3", feature = "azure", feature = "gcs", feature = "redis"))] #[async_trait] diff --git a/src/client.rs b/src/client.rs index e8780bd16..51a6a1fe3 100644 --- a/src/client.rs +++ b/src/client.rs @@ -81,9 +81,9 @@ pub fn connect_with_retry(port: u16) -> io::Result { // us once it starts the server instead of us polling. match retry(Fixed::from_millis(500).take(10), || connect_to_server(port)) { Ok(conn) => Ok(conn), - _ => Err(io::Error::new( + Err(e) => Err(io::Error::new( io::ErrorKind::TimedOut, - "Connection to server timed out", + format!("Connection to server timed out: {:?}", e), )), } } diff --git a/src/commands.rs b/src/commands.rs index 8805a157c..dc772488c 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -340,9 +340,9 @@ pub fn request_zero_stats(mut conn: ServerConnection) -> Result { /// Send a `GetStats` request to the server, and return the `ServerInfo` request if successful. pub fn request_stats(mut conn: ServerConnection) -> Result { debug!("request_stats"); - let response = conn - .request(Request::GetStats) - .context("Failed to send data to or receive data from server")?; + let response = conn.request(Request::GetStats).context( + "Failed to send data to or receive data from server. Mismatch of client/server versions?", + )?; if let Response::Stats(stats) = response { Ok(*stats) } else { diff --git a/src/compiler/args.rs b/src/compiler/args.rs index e551c0f80..64d7eb98a 100644 --- a/src/compiler/args.rs +++ b/src/compiler/args.rs @@ -575,6 +575,7 @@ where { arguments: I, arg_info: S, + seen_double_dashes: Option, phantom: PhantomData, } @@ -592,9 +593,15 @@ where ArgsIter { arguments, arg_info, + seen_double_dashes: None, phantom: PhantomData, } } + + pub fn with_double_dashes(mut self) -> Self { + self.seen_double_dashes = Some(false); + self + } } impl Iterator for ArgsIter @@ -607,6 +614,14 @@ where fn next(&mut self) -> Option { if let Some(arg) = self.arguments.next() { + if let Some(seen_double_dashes) = &mut self.seen_double_dashes { + if !*seen_double_dashes && arg == "--" { + *seen_double_dashes = true; + } + if *seen_double_dashes { + return Some(Ok(Argument::Raw(arg))); + } + } let s = arg.to_string_lossy(); let arguments = &mut self.arguments; Some(match self.arg_info.search(&s[..]) { @@ -957,8 +972,11 @@ mod tests { "-plop", "-quxbar", // -quxbar is not -qux with a value of bar "-qux=value", + "--", + "non_flag", + "-flag-after-double-dashes", ]; - let iter = ArgsIter::new(args.iter().map(OsString::from), &ARGS[..]); + let iter = ArgsIter::new(args.iter().map(OsString::from), &ARGS[..]).with_double_dashes(); let expected = vec![ arg!(UnknownFlag("-nomatch")), arg!(WithValue("-foo", ArgData::Foo("value"), Separated)), @@ -979,6 +997,9 @@ mod tests { ArgData::Qux("value"), CanBeSeparated('=') )), + arg!(Raw("--")), + arg!(Raw("non_flag")), + arg!(Raw("-flag-after-double-dashes")), ]; match diff_with(iter, expected, |a, b| { assert_eq!(a.as_ref().unwrap(), b); diff --git a/src/compiler/c.rs b/src/compiler/c.rs index 424105d5e..3aa9ea5a1 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -85,6 +85,8 @@ pub struct ArtifactDescriptor { pub struct ParsedArguments { /// The input source file. pub input: PathBuf, + /// Whether to prepend the input with `--` + pub double_dash_input: bool, /// The type of language used in the input source file. pub language: Language, /// The flag required to compile for the given language @@ -551,7 +553,7 @@ fn process_preprocessed_file( let mut normalized_include_paths: HashMap, Option>> = HashMap::new(); // There must be at least 7 characters (# 1 "x") left to potentially find an // include file path. - while start < total_len - 7 { + while start < total_len.saturating_sub(7) { let mut slice = &bytes[start..]; // Check if we look at a line containing the file name of an included file. // At least the following formats exist (where N is a positive integer): @@ -609,9 +611,12 @@ fn process_preprocessed_file( continue; } }; - } else if &bytes[start..start + INCBIN_DIRECTIVE.len()] == INCBIN_DIRECTIVE - && ((slice[7] == b' ' && (slice[8] == b'"' || (slice[8] == b'\\' && slice[9] == b'"'))) - || slice[7] == b'"') + } else if slice + .strip_prefix(INCBIN_DIRECTIVE) + .filter(|slice| { + slice.starts_with(b"\"") || slice.starts_with(b" \"") || slice.starts_with(b" \\\"") + }) + .is_some() { // An assembler .inc bin (without the space) statement, which could be // part of inline assembly, refers to an external file. If the file @@ -668,7 +673,7 @@ fn process_preprocessor_line( ) -> Result { let mut slice = &bytes[start..]; // Workarounds for preprocessor linemarker bugs in GCC version 6. - if slice[2] == b'3' { + if slice.get(2) == Some(&b'3') { if slice.starts_with(HASH_31_COMMAND_LINE_NEWLINE) { // Bogus extra line with #31, after the regular #1: // Ignore the whole line, and continue parsing. @@ -684,7 +689,7 @@ fn process_preprocessor_line( // Replace the line number with the usual one. digest.update(&bytes[hash_start..start]); start += 1; - bytes[start..start + 2].copy_from_slice(b"# 1"); + bytes[start..=start + 2].copy_from_slice(b"# 1"); hash_start = start; slice = &bytes[start..]; } diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index bd33f1c85..a5768be93 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -1381,7 +1381,7 @@ where mod test { use super::*; use crate::cache::disk::DiskCache; - use crate::cache::CacheRead; + use crate::cache::{CacheRead, PreprocessorCacheModeConfig}; use crate::mock_command::*; use crate::test::mock_storage::MockStorage; use crate::test::utils::*; @@ -1390,6 +1390,7 @@ mod test { use std::sync::Arc; use std::time::Duration; use std::u64; + use test_case::test_case; use tokio::runtime::Runtime; #[test] @@ -1669,14 +1670,17 @@ LLVM version: 6.0", .is_err()); } - #[test] - fn test_compiler_version_affects_hash() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_version_affects_hash(preprocessor_cache_mode: bool) { let f = TestFixture::new(); let creator = new_creator(); let runtime = single_threaded_runtime(); let pool = runtime.handle(); let arguments = ovec!["-c", "foo.c", "-o", "foo.o"]; let cwd = f.tempdir.path(); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let results: Vec<_> = [11, 12] .iter() @@ -1711,7 +1715,7 @@ LLVM version: 6.0", false, pool, false, - Arc::new(MockStorage::new(None)), + Arc::new(MockStorage::new(None, preprocessor_cache_mode)), CacheControl::Default, ) .wait() @@ -1741,8 +1745,9 @@ LLVM version: 6.0", assert_eq!(CompilerKind::C(CCompilerKind::Gcc), c.kind()); } - #[test] - fn test_compiler_get_cached_or_compile() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_get_cached_or_compile(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -1752,8 +1757,13 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let storage = Arc::new(storage); // Pretend to be GCC. next_command( @@ -1859,9 +1869,10 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] #[cfg(feature = "dist-client")] - fn test_compiler_get_cached_or_compile_dist() { + fn test_compiler_get_cached_or_compile_dist(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -1871,8 +1882,13 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let storage = Arc::new(storage); // Pretend to be GCC. next_command( @@ -1973,17 +1989,20 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] /// Test that a cache read that results in an error is treated as a cache /// miss. - fn test_compiler_get_cached_or_compile_cache_error() { + fn test_compiler_get_cached_or_compile_cache_error(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); let runtime = Runtime::new().unwrap(); let pool = runtime.handle().clone(); - let storage = MockStorage::new(None); + let storage = MockStorage::new(None, preprocessor_cache_mode); let storage: Arc = Arc::new(storage); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); // Pretend to be GCC. next_command( &creator, @@ -2056,17 +2075,20 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] /// Test that cache read timing is recorded. - fn test_compiler_get_cached_or_compile_cache_get_timing() { + fn test_compiler_get_cached_or_compile_cache_get_timing(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); let runtime = Runtime::new().unwrap(); let pool = runtime.handle().clone(); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); // Make our storage wait 2ms for each get/put operation. let storage_delay = Duration::from_millis(2); - let storage = MockStorage::new(Some(storage_delay)); + let storage = MockStorage::new(Some(storage_delay), preprocessor_cache_mode); let storage: Arc = Arc::new(storage); // Pretend to be GCC. next_command( @@ -2135,8 +2157,9 @@ LLVM version: 6.0", } } - #[test] - fn test_compiler_get_cached_or_compile_force_recache() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_get_cached_or_compile_force_recache(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -2146,9 +2169,14 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); let storage = Arc::new(storage); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); // Pretend to be GCC. next_command( &creator, @@ -2254,8 +2282,9 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] - fn test_compiler_get_cached_or_compile_preprocessor_error() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_get_cached_or_compile_preprocessor_error(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -2265,12 +2294,17 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); let storage = Arc::new(storage); // Pretend to be GCC. Also inject a fake object file that the subsequent // preprocessor failure should remove. let obj = f.tempdir.path().join("foo.o"); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let o = obj.clone(); next_command_calls(&creator, move |_| { let mut f = File::create(&o)?; @@ -2332,9 +2366,10 @@ LLVM version: 6.0", assert!(fs::metadata(&obj).is_err()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] #[cfg(feature = "dist-client")] - fn test_compiler_get_cached_or_compile_dist_error() { + fn test_compiler_get_cached_or_compile_dist_error(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -2346,11 +2381,16 @@ LLVM version: 6.0", test_dist::ErrorSubmitToolchainClient::new(), test_dist::ErrorRunJobClient::new(), ]; + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let storage = DiskCache::new( f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); let storage = Arc::new(storage); // Pretend to be GCC. diff --git a/src/compiler/diab.rs b/src/compiler/diab.rs index 307ff17c0..7ef453b29 100644 --- a/src/compiler/diab.rs +++ b/src/compiler/diab.rs @@ -285,6 +285,7 @@ where CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input: false, language, compilation_flag, depfile: None, @@ -351,9 +352,9 @@ pub fn generate_compile_commands( "-o".into(), out_file.into(), ]; - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); let command = CompileCommand { executable: executable.to_owned(), arguments, @@ -744,6 +745,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index e422d8735..0bd2e224b 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -254,6 +254,7 @@ where { let mut output_arg = None; let mut input_arg = None; + let mut double_dash_input = false; let mut dep_target = None; let mut dep_flag = OsString::from("-MT"); let mut common_args = vec![]; @@ -290,7 +291,11 @@ where let mut too_hard_for_preprocessor_cache_mode = false; - for arg in ArgsIter::new(it, arg_info) { + let mut args_iter = ArgsIter::new(it, arg_info); + if kind == CCompilerKind::Clang { + args_iter = args_iter.with_double_dashes(); + } + for arg in args_iter { let arg = try_or_cannot_cache!(arg, "argument parse"); // Check if the value part of this argument begins with '@'. If so, we either // failed to expand it, or it was a concatenated argument - either way, bail. @@ -392,6 +397,11 @@ where } Some(XClang(s)) => xclangs.push(s.clone()), None => match arg { + Argument::Raw(ref val) if val == "--" => { + if input_arg.is_none() { + double_dash_input = true; + } + } Argument::Raw(ref val) => { if input_arg.is_some() { multiple_input = true; @@ -605,6 +615,7 @@ where CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input, language, compilation_flag, depfile: None, @@ -703,11 +714,14 @@ fn preprocess_cmd( arch_args_to_use = &parsed_args.arch_args; } - cmd.arg(&parsed_args.input) - .args(&parsed_args.preprocessor_args) + cmd.args(&parsed_args.preprocessor_args) .args(&parsed_args.dependency_args) .args(&parsed_args.common_args) - .args(arch_args_to_use) + .args(arch_args_to_use); + if parsed_args.double_dash_input { + cmd.arg("--"); + } + cmd.arg(&parsed_args.input) .env_clear() .envs(env_vars.iter().map(|(k, v)| (k, v))) .current_dir(cwd); @@ -780,14 +794,17 @@ pub fn generate_compile_commands( } arguments.extend(vec![ parsed_args.compilation_flag.clone(), - parsed_args.input.clone().into(), "-o".into(), out_file.into(), ]); - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); - arguments.extend(parsed_args.arch_args.clone()); + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); + arguments.extend_from_slice(&parsed_args.arch_args); + if parsed_args.double_dash_input { + arguments.push("--".into()); + } + arguments.push(parsed_args.input.clone().into()); let command = CompileCommand { executable: executable.to_owned(), arguments, @@ -931,7 +948,7 @@ mod test { use crate::mock_command::*; use crate::test::utils::*; - use temp_env::with_var; + use temp_env::{with_var, with_var_unset}; fn parse_arguments_( arguments: Vec, @@ -941,6 +958,20 @@ mod test { parse_arguments(&args, ".".as_ref(), &ARGS[..], plusplus, CCompilerKind::Gcc) } + fn parse_arguments_clang( + arguments: Vec, + plusplus: bool, + ) -> CompilerArguments { + let args = arguments.iter().map(OsString::from).collect::>(); + parse_arguments( + &args, + ".".as_ref(), + &ARGS[..], + plusplus, + CCompilerKind::Clang, + ) + } + #[test] fn test_parse_arguments_simple() { let args = stringvec!["-c", "foo.c", "-o", "foo.o"]; @@ -1336,6 +1367,65 @@ mod test { assert!(!msvc_show_includes); } + #[test] + fn test_parse_arguments_double_dash() { + let args = stringvec!["-c", "-o", "foo.o", "--", "foo.c"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_(args.clone(), false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // GCC doesn't support double dashes. If we got one, we'll pass them + // through to GCC for it to error out. + assert!(!double_dash_input); + assert_eq!(ovec!["--"], common_args); + + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + assert!(double_dash_input); + assert!(common_args.is_empty()); + + let args = stringvec!["-c", "-o", "foo.o", "foo.c", "--"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // Double dash after input file is ignored. + assert!(!double_dash_input); + assert!(common_args.is_empty()); + + let args = stringvec!["-c", "-o", "foo.o", "foo.c", "--", "bar.c"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"bar.c\"]".to_string())), + parse_arguments_clang(args, false) + ); + + let args = stringvec!["-c", "-o", "foo.o", "foo.c", "--", "-fPIC"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"-fPIC\"]".to_string())), + parse_arguments_clang(args, false) + ); + } + #[test] fn test_parse_arguments_explicit_dep_target() { let args = @@ -1503,9 +1593,9 @@ mod test { "c++", "-E", "-fdirectives-only", - "foo.cc", "-D__arm64__=1", - "-D__i386__=1" + "-D__i386__=1", + "foo.cc" ]; assert_eq!(cmd.args, expected_args); }); @@ -1538,13 +1628,38 @@ mod test { "c++", "-E", "-fdirectives-only", - "foo.cc", "-arch", - "arm64" + "arm64", + "foo.cc" ]; assert_eq!(cmd.args, expected_args); } + #[test] + fn test_preprocess_double_dash_input() { + let args = stringvec!["-c", "-o", "foo.o", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let mut cmd = MockCommand { + child: None, + args: vec![], + }; + preprocess_cmd( + &mut cmd, + &parsed_args, + Path::new(""), + &[], + true, + CCompilerKind::Clang, + true, + vec![], + ); + let expected_args = ovec!["-x", "c", "-E", "-frewrite-includes", "--", "foo.c"]; + assert_eq!(cmd.args, expected_args); + } + #[test] fn pedantic_default() { let args = stringvec!["-pedantic", "-c", "foo.cc"]; @@ -1768,18 +1883,20 @@ mod test { #[test] fn test_parse_arguments_multiarch_cache_disabled() { - assert_eq!( - CompilerArguments::CannotCache( - "multiple different -arch, and SCCACHE_CACHE_MULTIARCH not set", - None - ), - parse_arguments_( - stringvec![ - "-fPIC", "-arch", "arm64", "-arch", "i386", "-o", "foo.o", "-c", "foo.cpp" - ], - false + with_var_unset("SCCACHE_CACHE_MULTIARCH", || { + assert_eq!( + CompilerArguments::CannotCache( + "multiple different -arch, and SCCACHE_CACHE_MULTIARCH not set", + None + ), + parse_arguments_( + stringvec![ + "-fPIC", "-arch", "arm64", "-arch", "i386", "-o", "foo.o", "-c", "foo.cpp" + ], + false + ) ) - ) + }); } #[test] @@ -1888,6 +2005,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, @@ -1936,6 +2054,30 @@ mod test { assert_eq!(0, creator.lock().unwrap().children.len()); } + #[test] + fn test_compile_double_dash_input() { + let args = stringvec!["-c", "-o", "foo.o", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let f = TestFixture::new(); + let compiler = &f.bins[0]; + let mut path_transformer = dist::PathTransformer::default(); + let (command, _, _) = generate_compile_commands( + &mut path_transformer, + compiler, + &parsed_args, + f.tempdir.path(), + &[], + CCompilerKind::Clang, + false, + ) + .unwrap(); + let expected_args = ovec!["-x", "c", "-c", "-o", "foo.o", "--", "foo.c"]; + assert_eq!(command.arguments, expected_args); + } + #[test] fn test_parse_arguments_plusplus() { let args = stringvec!["-c", "foo.c", "-o", "foo.o"]; diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index e6474fd9d..56b2260d2 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -492,6 +492,7 @@ pub fn parse_arguments( ) -> CompilerArguments { let mut output_arg = None; let mut input_arg = None; + let mut double_dash_input = false; let mut common_args = vec![]; let mut unhashed_args = vec![]; let mut preprocessor_args = vec![]; @@ -512,7 +513,10 @@ pub fn parse_arguments( // Custom iterator to expand `@` arguments which stand for reading a file // and interpreting it as a list of more arguments. let it = ExpandIncludeFile::new(cwd, arguments); - let it = ArgsIter::new(it, (&ARGS[..], &SLASH_ARGS[..])); + let mut it = ArgsIter::new(it, (&ARGS[..], &SLASH_ARGS[..])); + if is_clang { + it = it.with_double_dashes(); + } for arg in it { let arg = try_or_cannot_cache!(arg, "argument parse"); match arg.get_data() { @@ -553,6 +557,11 @@ pub fn parse_arguments( Some(Clang(s)) => clangs.push(s.clone()), None => { match arg { + Argument::Raw(ref val) if val == "--" => { + if input_arg.is_none() { + double_dash_input = true; + } + } Argument::Raw(ref val) => { if input_arg.is_some() { // Can't cache compilations with multiple inputs. @@ -797,6 +806,7 @@ pub fn parse_arguments( CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input, language, compilation_flag, depfile, @@ -852,22 +862,17 @@ fn normpath(path: &str) -> String { } #[allow(clippy::too_many_arguments)] -pub async fn preprocess( - creator: &T, - executable: &Path, +pub fn preprocess_cmd( + cmd: &mut T, parsed_args: &ParsedArguments, cwd: &Path, env_vars: &[(OsString, OsString)], may_dist: bool, - includes_prefix: &str, rewrite_includes_only: bool, is_clang: bool, -) -> Result -where - T: CommandCreatorSync, +) where + T: RunCommand, { - let mut cmd = creator.clone().new_command_sync(executable); - // When performing distributed compilation, line number info is important for error // reporting and to not cause spurious compilation failure (e.g. no exceptions build // fails due to exceptions transitively included in the stdlib). @@ -880,14 +885,10 @@ where cmd.arg("-EP"); } - cmd.arg(&parsed_args.input) - .arg("-nologo") + cmd.arg("-nologo") .args(&parsed_args.preprocessor_args) .args(&parsed_args.dependency_args) .args(&parsed_args.common_args) - // Windows SDK generates C4668 during preprocessing, but compiles fine. - // Read for more info: https://github.com/mozilla/sccache/issues/1725 - .arg("/wd4668") .env_clear() .envs(env_vars.iter().map(|(k, v)| (k, v))) .current_dir(cwd); @@ -902,12 +903,47 @@ where cmd.arg("/sourceDependencies"); cmd.arg(depfile); } + // Windows SDK generates C4668 during preprocessing, but compiles fine. + // Read for more info: https://github.com/mozilla/sccache/issues/1725 + cmd.arg("/wd4668"); } if rewrite_includes_only && is_clang { cmd.arg("-clang:-frewrite-includes"); } + if parsed_args.double_dash_input { + cmd.arg("--"); + } + cmd.arg(&parsed_args.input); +} + +#[allow(clippy::too_many_arguments)] +pub async fn preprocess( + creator: &T, + executable: &Path, + parsed_args: &ParsedArguments, + cwd: &Path, + env_vars: &[(OsString, OsString)], + may_dist: bool, + includes_prefix: &str, + rewrite_includes_only: bool, + is_clang: bool, +) -> Result +where + T: CommandCreatorSync, +{ + let mut cmd = creator.clone().new_command_sync(executable); + preprocess_cmd( + &mut cmd, + parsed_args, + cwd, + env_vars, + may_dist, + rewrite_includes_only, + is_clang, + ); + if log_enabled!(Debug) { debug!("preprocess: {:?}", cmd); } @@ -1013,16 +1049,15 @@ fn generate_compile_commands( let mut fo = OsString::from("-Fo"); fo.push(out_file); - let mut arguments: Vec = vec![ - parsed_args.compilation_flag.clone(), - parsed_args.input.clone().into(), - fo, - ]; - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.dependency_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); - + let mut arguments: Vec = vec![parsed_args.compilation_flag.clone(), fo]; + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.dependency_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); + if parsed_args.double_dash_input { + arguments.push("--".into()); + } + arguments.push(parsed_args.input.clone().into()); let command = CompileCommand { executable: executable.to_owned(), arguments, @@ -1039,17 +1074,19 @@ fn generate_compile_commands( let mut fo = String::from("-Fo"); fo.push_str(&path_transformer.as_dist(out_file)?); - let mut arguments: Vec = vec![ - parsed_args.compilation_flag.clone().into_string().ok()?, - path_transformer.as_dist(&parsed_args.input)?, - fo, - ]; + let mut arguments: Vec = + vec![parsed_args.compilation_flag.clone().into_string().ok()?, fo]; // It's important to avoid preprocessor_args because of things like /FI which // forcibly includes another file. This does mean we're potentially vulnerable // to misidentification of flags like -DYNAMICBASE (though in that specific // case we're safe as it only applies to link time, which sccache avoids). arguments.extend(dist::osstrings_to_strings(&parsed_args.common_args)?); + if parsed_args.double_dash_input { + arguments.push("--".into()); + } + arguments.push(path_transformer.as_dist(&parsed_args.input)?); + Some(dist::CompileCommand { executable: path_transformer.as_dist(executable)?, arguments, @@ -1131,7 +1168,7 @@ impl<'a> Iterator for ExpandIncludeFile<'a> { loop { // Visit all arguments found in the most recently read response file. // Since response files are not recursive, we do not need to worry - // about these containing addditional @ directives. + // about these containing additional @ directives. if let Some(response_file_arg) = self.stack.pop() { return Some(response_file_arg); } @@ -1151,7 +1188,7 @@ impl<'a> Iterator for ExpandIncludeFile<'a> { Ok(content) => content, Err(err) => { debug!("failed to read @-file `{}`: {}", file_path.display(), err); - // If we failed to read the file content, return the orginal arg (including the `@` directive). + // If we failed to read the file content, return the original arg (including the `@` directive). return Some(arg); } }; @@ -1321,6 +1358,10 @@ mod test { super::parse_arguments(&arguments, &std::env::current_dir().unwrap(), false) } + fn parse_arguments_clang(arguments: Vec) -> CompilerArguments { + super::parse_arguments(&arguments, &std::env::current_dir().unwrap(), true) + } + #[test] fn test_detect_showincludes_prefix() { drop(env_logger::try_init()); @@ -1510,6 +1551,65 @@ mod test { assert!(!msvc_show_includes); } + #[test] + fn test_parse_arguments_double_dash() { + let args = ovec!["-c", "-Fofoo.obj", "--", "foo.c"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments(args.clone()) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // MSVC doesn't support double dashes. If we got one, we'll pass them + // through to MSVC for it to error out. + assert!(!double_dash_input); + assert_eq!(ovec!["--"], common_args); + + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + assert!(double_dash_input); + assert!(common_args.is_empty()); + + let args = ovec!["-c", "-Fofoo.obj", "foo.c", "--"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // Double dash after input file is ignored. + assert!(!double_dash_input); + assert!(common_args.is_empty()); + + let args = ovec!["-c", "-Fofoo.obj", "foo.c", "--", "bar.c"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"bar.c\"]".to_string())), + parse_arguments_clang(args) + ); + + let args = ovec!["-c", "-Fofoo.obj", "foo.c", "--", "-fPIC"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"-fPIC\"]".to_string())), + parse_arguments_clang(args) + ); + } + #[test] fn parse_argument_slashes() { let args = ovec!["-c", "foo.c", "/Fofoo.obj"]; @@ -2241,12 +2341,29 @@ mod test { ); } + #[test] + fn test_preprocess_double_dash_input() { + let args = ovec!["-c", "-Fofoo.o.bj", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let mut cmd = MockCommand { + child: None, + args: vec![], + }; + preprocess_cmd(&mut cmd, &parsed_args, Path::new(""), &[], true, true, true); + let expected_args = ovec!["-E", "-nologo", "-clang:-frewrite-includes", "--", "foo.c"]; + assert_eq!(cmd.args, expected_args); + } + #[test] fn test_compile_simple() { let creator = new_creator(); let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, @@ -2293,6 +2410,28 @@ mod test { assert_eq!(0, creator.lock().unwrap().children.len()); } + #[test] + fn test_compile_double_dash_input() { + let args = ovec!["-c", "-Fofoo.obj", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let f = TestFixture::new(); + let compiler = &f.bins[0]; + let mut path_transformer = dist::PathTransformer::default(); + let (command, _, _) = generate_compile_commands( + &mut path_transformer, + compiler, + &parsed_args, + f.tempdir.path(), + &[], + ) + .unwrap(); + let expected_args = ovec!["-c", "-Fofoo.obj", "--", "foo.c"]; + assert_eq!(command.arguments, expected_args); + } + #[test] fn test_compile_not_cacheable_pdb() { let creator = new_creator(); @@ -2300,6 +2439,7 @@ mod test { let pdb = f.touch("foo.pdb").unwrap(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "/c".into(), depfile: None, diff --git a/src/compiler/nvhpc.rs b/src/compiler/nvhpc.rs index e86ce782e..a09eb6cac 100644 --- a/src/compiler/nvhpc.rs +++ b/src/compiler/nvhpc.rs @@ -132,7 +132,7 @@ impl CCompilerImpl for Nvhpc { trace!("preprocess"); let mut cmd = initialize_cmd_and_args(); - //NVHPC doesn't support disabling line info when outputing to console + //NVHPC doesn't support disabling line info when outputting to console cmd.arg("-E") .env_clear() .envs(env_vars.iter().map(|(k, v)| (k, v))) diff --git a/src/compiler/preprocessor_cache.rs b/src/compiler/preprocessor_cache.rs index d7adaacd0..3d263348b 100644 --- a/src/compiler/preprocessor_cache.rs +++ b/src/compiler/preprocessor_cache.rs @@ -416,7 +416,8 @@ pub fn preprocessor_cache_entry_hash_key( let mut buf = vec![]; encode_path(&mut buf, input_file)?; m.update(&buf); - let reader = std::fs::File::open(input_file).context("while hashing the input file")?; + let reader = std::fs::File::open(input_file) + .with_context(|| format!("while hashing the input file '{}'", input_file.display()))?; let digest = if config.ignore_time_macros { Digest::reader_sync(reader)? diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index d97113657..dca3ddf9e 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -2435,6 +2435,7 @@ mod test { use std::ffi::OsStr; use std::io::{self, Write}; use std::sync::{Arc, Mutex}; + use test_case::test_case; fn _parse_arguments(arguments: &[String]) -> CompilerArguments { let arguments = arguments.iter().map(OsString::from).collect::>(); @@ -3076,8 +3077,9 @@ proc_macro false ); } - #[test] - fn test_generate_hash_key() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_generate_hash_key(preprocessor_cache_mode: bool) { use ar::{Builder, Header}; drop(env_logger::try_init()); let f = TestFixture::new(); @@ -3160,7 +3162,7 @@ proc_macro false false, &pool, false, - Arc::new(MockStorage::new(None)), + Arc::new(MockStorage::new(None, preprocessor_cache_mode)), CacheControl::Default, ) .wait() @@ -3205,6 +3207,7 @@ proc_macro false args: &[&'static str], env_vars: &[(OsString, OsString)], pre_func: F, + preprocessor_cache_mode: bool, ) -> String where F: Fn(&Path) -> Result<()>, @@ -3251,7 +3254,7 @@ proc_macro false false, &pool, false, - Arc::new(MockStorage::new(None)), + Arc::new(MockStorage::new(None, preprocessor_cache_mode)), CacheControl::Default, ) .wait() @@ -3264,8 +3267,9 @@ proc_macro false Ok(()) } - #[test] - fn test_equal_hashes_externs() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_externs(preprocessor_cache_mode: bool) { // Put some content in the extern rlibs so we can verify that the content hashes are // used in the right order. fn mk_files(tempdir: &Path) -> Result<()> { @@ -3293,7 +3297,8 @@ proc_macro false "b=b.rlib" ], &[], - mk_files + mk_files, + preprocessor_cache_mode, ), hash_key( &f, @@ -3313,13 +3318,15 @@ proc_macro false "lib" ], &[], - mk_files + mk_files, + preprocessor_cache_mode, ) ); } - #[test] - fn test_equal_hashes_link_paths() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_link_paths(preprocessor_cache_mode: bool) { let f = TestFixture::new(); assert_eq!( hash_key( @@ -3340,7 +3347,8 @@ proc_macro false "y=y" ], &[], - nothing + nothing, + preprocessor_cache_mode, ), hash_key( &f, @@ -3360,13 +3368,15 @@ proc_macro false "lib" ], &[], - nothing + nothing, + preprocessor_cache_mode, ) ); } - #[test] - fn test_equal_hashes_ignored_args() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_ignored_args(preprocessor_cache_mode: bool) { let f = TestFixture::new(); assert_eq!( hash_key( @@ -3389,7 +3399,8 @@ proc_macro false "y=y" ], &[], - nothing + nothing, + preprocessor_cache_mode, ), hash_key( &f, @@ -3411,13 +3422,15 @@ proc_macro false "lib" ], &[], - nothing + nothing, + preprocessor_cache_mode, ) ); } - #[test] - fn test_equal_hashes_cfg_features() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_cfg_features(preprocessor_cache_mode: bool) { let f = TestFixture::new(); assert_eq!( hash_key( @@ -3438,7 +3451,8 @@ proc_macro false "feature=b" ], &[], - nothing + nothing, + preprocessor_cache_mode, ), hash_key( &f, @@ -3458,7 +3472,8 @@ proc_macro false "lib" ], &[], - nothing + nothing, + preprocessor_cache_mode, ) ); } diff --git a/src/compiler/tasking_vx.rs b/src/compiler/tasking_vx.rs index 1bd4a02a1..09cc8d74a 100644 --- a/src/compiler/tasking_vx.rs +++ b/src/compiler/tasking_vx.rs @@ -266,6 +266,7 @@ where CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input: false, language, compilation_flag: "-c".into(), depfile, @@ -366,9 +367,9 @@ fn generate_compile_commands( "-o".into(), out_file.path.as_os_str().into(), ]; - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); let command = CompileCommand { executable: executable.to_owned(), arguments, @@ -683,6 +684,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, @@ -731,6 +733,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.cu".into(), + double_dash_input: false, language: Language::Cuda, compilation_flag: "-c".into(), depfile: None, diff --git a/src/config.rs b/src/config.rs index c8614d31e..a88a61e0d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -33,10 +33,11 @@ use std::result::Result as StdResult; use std::str::FromStr; use std::sync::Mutex; -use crate::{cache::PreprocessorCacheModeConfig, errors::*}; +pub use crate::cache::PreprocessorCacheModeConfig; +use crate::errors::*; static CACHED_CONFIG_PATH: Lazy = Lazy::new(CachedConfig::file_config_path); -static CACHED_CONFIG: Lazy>> = Lazy::new(|| Mutex::new(None)); +static CACHED_CONFIG: Mutex> = Mutex::new(None); const ORGANIZATION: &str = "Mozilla"; const APP_NAME: &str = "sccache"; @@ -168,7 +169,7 @@ impl Default for DiskCacheConfig { DiskCacheConfig { dir: default_disk_cache_dir(), size: default_disk_cache_size(), - preprocessor_cache_mode: Default::default(), + preprocessor_cache_mode: PreprocessorCacheModeConfig::activated(), } } } @@ -602,7 +603,7 @@ fn config_from_env() -> Result { if env::var("SCCACHE_GCS_OAUTH_URL").is_ok() { eprintln!("SCCACHE_GCS_OAUTH_URL has been deprecated"); - eprintln!("if you intend to use vm metadata for auth, please set correct service account intead"); + eprintln!("if you intend to use vm metadata for auth, please set correct service account instead"); } let credential_url = env::var("SCCACHE_GCS_CREDENTIALS_URL").ok(); @@ -709,14 +710,21 @@ fn config_from_env() -> Result { .ok() .and_then(|v| parse_size(&v)); - let mut preprocessor_mode_config = PreprocessorCacheModeConfig::default(); - match env::var("SCCACHE_DIRECT").as_deref() { - Ok("on") | Ok("true") => preprocessor_mode_config.use_preprocessor_cache_mode = true, - Ok("off") | Ok("false") => preprocessor_mode_config.use_preprocessor_cache_mode = false, - _ => {} + let mut preprocessor_mode_config = PreprocessorCacheModeConfig::activated(); + let preprocessor_mode_overridden = match env::var("SCCACHE_DIRECT").as_deref() { + Ok("on") | Ok("true") => { + preprocessor_mode_config.use_preprocessor_cache_mode = true; + true + } + Ok("off") | Ok("false") => { + preprocessor_mode_config.use_preprocessor_cache_mode = false; + true + } + _ => false, }; - let disk = if disk_dir.is_some() || disk_sz.is_some() { + let any_overridden = disk_dir.is_some() || disk_sz.is_some() || preprocessor_mode_overridden; + let disk = if any_overridden { Some(DiskCacheConfig { dir: disk_dir.unwrap_or_else(default_disk_cache_dir), size: disk_sz.unwrap_or_else(default_disk_cache_size), @@ -1288,7 +1296,7 @@ token = "webdavtoken" disk: Some(DiskCacheConfig { dir: PathBuf::from("/tmp/.cache/sccache"), size: 7 * 1024 * 1024 * 1024, - preprocessor_cache_mode: Default::default(), + preprocessor_cache_mode: PreprocessorCacheModeConfig::activated(), }), gcs: Some(GCSCacheConfig { bucket: "bucket".to_owned(), diff --git a/src/dist/client_auth.rs b/src/dist/client_auth.rs index 573de1dea..cfe506106 100644 --- a/src/dist/client_auth.rs +++ b/src/dist/client_auth.rs @@ -89,7 +89,6 @@ mod code_grant_pkce { use base64::Engine; use futures::channel::oneshot; use hyper::{Body, Method, Request, Response, StatusCode}; - use once_cell::sync::Lazy; use rand::{rngs::OsRng, RngCore}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -142,7 +141,7 @@ mod code_grant_pkce { pub shutdown_tx: Option>, } - pub static STATE: Lazy>> = Lazy::new(|| Mutex::new(None)); + pub static STATE: Mutex> = Mutex::new(None); pub fn generate_verifier_and_challenge() -> Result<(String, String)> { let mut code_verifier_bytes = vec![0; NUM_CODE_VERIFIER_BYTES]; @@ -279,7 +278,6 @@ mod implicit { }; use futures::channel::oneshot; use hyper::{Body, Method, Request, Response, StatusCode}; - use once_cell::sync::Lazy; use std::collections::HashMap; use std::sync::mpsc; use std::sync::Mutex; @@ -308,7 +306,7 @@ mod implicit { pub shutdown_tx: Option>, } - pub static STATE: Lazy>> = Lazy::new(|| Mutex::new(None)); + pub static STATE: Mutex> = Mutex::new(None); pub fn finish_url(client_id: &str, url: &mut Url, redirect_uri: &str, state: &str) { url.query_pairs_mut() diff --git a/src/server.rs b/src/server.rs index 8237c3d50..562fc39ba 100644 --- a/src/server.rs +++ b/src/server.rs @@ -36,6 +36,7 @@ use futures::future::FutureExt; use futures::{future, stream, Sink, SinkExt, Stream, StreamExt, TryFutureExt}; use number_prefix::NumberPrefix; use serde::{Deserialize, Serialize}; +use std::cell::Cell; use std::collections::HashMap; use std::env; use std::ffi::{OsStr, OsString}; @@ -392,12 +393,29 @@ impl DistClientContainer { } } +thread_local! { + /// catch_unwind doesn't provide panic location, so we store that + /// information via a panic hook to be used when catch_unwind + /// catches a panic. + static PANIC_LOCATION: Cell> = Cell::new(None); +} + /// Start an sccache server, listening on `port`. /// /// Spins an event loop handling client connections until a client /// requests a shutdown. pub fn start_server(config: &Config, port: u16) -> Result<()> { info!("start_server: port: {}", port); + let panic_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + PANIC_LOCATION.with(|l| { + l.set( + info.location() + .map(|loc| (loc.file().to_string(), loc.line(), loc.column())), + ) + }); + panic_hook(info) + })); let client = unsafe { Client::new() }; let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -896,6 +914,10 @@ where async fn get_info(&self) -> Result { let stats = self.stats.lock().await.clone(); let cache_location = self.storage.location(); + let use_preprocessor_cache_mode = self + .storage + .preprocessor_cache_mode_config() + .use_preprocessor_cache_mode; let version = env!("CARGO_PKG_VERSION").to_string(); futures::try_join!(self.storage.current_size(), self.storage.max_size()).map( move |(cache_size, max_cache_size)| ServerInfo { @@ -903,6 +925,7 @@ where cache_location, cache_size, max_cache_size, + use_preprocessor_cache_mode, version, }, ) @@ -1177,20 +1200,35 @@ where let task = async move { let dist_client = me.dist_client.get_client().await; let result = match dist_client { - Ok(client) => { - hasher - .get_cached_or_compile( - client, - creator, - storage, - arguments, - cwd, - env_vars, - cache_control, - pool, + Ok(client) => std::panic::AssertUnwindSafe(hasher.get_cached_or_compile( + client, + creator, + storage, + arguments, + cwd, + env_vars, + cache_control, + pool, + )) + .catch_unwind() + .await + .map_err(|e| { + let panic = e + .downcast_ref::<&str>() + .map(|s| &**s) + .or_else(|| e.downcast_ref::().map(|s| &**s)) + .unwrap_or("An unknown panic was caught."); + let thread = std::thread::current(); + let thread_name = thread.name().unwrap_or("unnamed"); + if let Some((file, line, column)) = PANIC_LOCATION.with(|l| l.take()) { + anyhow!( + "thread '{thread_name}' panicked at {file}:{line}:{column}: {panic}" ) - .await - } + } else { + anyhow!("thread '{thread_name}' panicked: {panic}") + } + }) + .and_then(std::convert::identity), Err(e) => Err(e), }; let mut cache_write = None; @@ -1442,6 +1480,7 @@ pub struct ServerInfo { pub cache_location: String, pub cache_size: Option, pub max_cache_size: Option, + pub use_preprocessor_cache_mode: bool, pub version: String, } @@ -1651,6 +1690,18 @@ impl ServerInfo { self.cache_location, name_width = name_width ); + if self.cache_location.starts_with("Local disk") { + println!( + "{:>>>, tx: mpsc::UnboundedSender>, delay: Option, + preprocessor_cache_mode: bool, } impl MockStorage { /// Create a new `MockStorage`. if `delay` is `Some`, wait for that amount of time before returning from operations. - pub(crate) fn new(delay: Option) -> MockStorage { + pub(crate) fn new(delay: Option, preprocessor_cache_mode: bool) -> MockStorage { let (tx, rx) = mpsc::unbounded(); Self { tx, rx: Arc::new(Mutex::new(rx)), delay, + preprocessor_cache_mode, } } @@ -72,4 +74,10 @@ impl Storage for MockStorage { async fn max_size(&self) -> Result> { Ok(None) } + fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: self.preprocessor_cache_mode, + ..Default::default() + } + } } diff --git a/src/test/tests.rs b/src/test/tests.rs index 39a1317d4..54ad27f62 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -229,6 +229,8 @@ fn test_server_compile() { const STDOUT: &[u8] = b"some stdout"; const STDERR: &[u8] = b"some stderr"; let conn = connect_to_server(port).unwrap(); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("file.c"), "whatever").unwrap(); { let mut c = server_creator.lock().unwrap(); // The server will check the compiler. Pretend it's GCC. diff --git a/tests/dist.rs b/tests/dist.rs index 4885e0451..f9c0ec883 100644 --- a/tests/dist.rs +++ b/tests/dist.rs @@ -52,7 +52,7 @@ pub fn dist_test_sccache_client_cfg( tmpdir: &Path, scheduler_url: HTTPUrl, ) -> sccache::config::FileConfig { - let mut sccache_cfg = harness::sccache_client_cfg(tmpdir); + let mut sccache_cfg = harness::sccache_client_cfg(tmpdir, false); sccache_cfg.cache.disk.as_mut().unwrap().size = 0; sccache_cfg.dist.scheduler_url = Some(scheduler_url); sccache_cfg diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index f53890543..b46fc033d 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -43,16 +43,22 @@ const TC_CACHE_SIZE: u64 = 1024 * 1024 * 1024; // 1 gig pub fn start_local_daemon(cfg_path: &Path, cached_cfg_path: &Path) { // Don't run this with run() because on Windows `wait_with_output` // will hang because the internal server process is not detached. - let _ = sccache_command() + if !sccache_command() .arg("--start-server") // Uncomment following lines to debug locally. - .env("SCCACHE_LOG", "debug") - .env("SCCACHE_ERROR_LOG", "/tmp/sccache_local_daemon.txt") + // .env("SCCACHE_LOG", "debug") + // .env( + // "SCCACHE_ERROR_LOG", + // env::temp_dir().join("sccache_local_daemon.txt"), + // ) .env("SCCACHE_CONF", cfg_path) .env("SCCACHE_CACHED_CONF", cached_cfg_path) .status() .unwrap() - .success(); + .success() + { + panic!("Failed to start local daemon"); + } } pub fn stop_local_daemon() { @@ -122,7 +128,10 @@ pub fn sccache_dist_path() -> PathBuf { assert_cmd::cargo::cargo_bin("sccache-dist") } -pub fn sccache_client_cfg(tmpdir: &Path) -> sccache::config::FileConfig { +pub fn sccache_client_cfg( + tmpdir: &Path, + preprocessor_cache_mode: bool, +) -> sccache::config::FileConfig { let cache_relpath = "client-cache"; let dist_cache_relpath = "client-dist-cache"; fs::create_dir(tmpdir.join(cache_relpath)).unwrap(); @@ -130,6 +139,10 @@ pub fn sccache_client_cfg(tmpdir: &Path) -> sccache::config::FileConfig { let disk_cache = sccache::config::DiskCacheConfig { dir: tmpdir.join(cache_relpath), + preprocessor_cache_mode: sccache::config::PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ..Default::default() }; sccache::config::FileConfig { @@ -154,6 +167,7 @@ pub fn sccache_client_cfg(tmpdir: &Path) -> sccache::config::FileConfig { server_startup_timeout_ms: None, } } + #[cfg(feature = "dist-server")] fn sccache_scheduler_cfg() -> sccache::config::scheduler::Config { sccache::config::scheduler::Config { @@ -164,6 +178,7 @@ fn sccache_scheduler_cfg() -> sccache::config::scheduler::Config { }, } } + #[cfg(feature = "dist-server")] fn sccache_server_cfg( tmpdir: &Path, diff --git a/tests/system.rs b/tests/system.rs index abf890fb0..c7716de25 100644 --- a/tests/system.rs +++ b/tests/system.rs @@ -38,6 +38,8 @@ use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::str; +use std::time::{Duration, SystemTime}; +use test_case::test_case; use which::{which, which_in}; mod harness; @@ -151,6 +153,13 @@ fn copy_to_tempdir(inputs: &[&str], tempdir: &Path) { let source_file = tempdir.join(f); trace!("fs::copy({:?}, {:?})", original_source_file, source_file); fs::copy(&original_source_file, &source_file).unwrap(); + // Preprocessor cache will not cache files that are too recent. + // Certain OS/FS combinations have a slow resolution (up to 2s for NFS), + // leading to flaky tests. + // We set the times for the new file to 10 seconds ago, to be safe. + let new_time = + filetime::FileTime::from_system_time(SystemTime::now() - Duration::from_secs(10)); + filetime::set_file_times(source_file, new_time, new_time).unwrap(); } } @@ -450,7 +459,7 @@ fn test_compile_with_define(compiler: Compiler, tempdir: &Path) { .stderr(predicates::str::contains("warning:").from_utf8().not()); } -fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path) { +fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path, preprocessor_cache_mode: bool) { if compiler.name != "clang++" { test_basic_compile(compiler.clone(), tempdir); } @@ -495,9 +504,19 @@ fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path) { version_output ), }; - test_clang_cache_whitespace_normalization(compiler, tempdir, !is_appleclang && major >= 14); + test_clang_cache_whitespace_normalization( + compiler, + tempdir, + !is_appleclang && major >= 14, + preprocessor_cache_mode, + ); } else { - test_clang_cache_whitespace_normalization(compiler, tempdir, false); + test_clang_cache_whitespace_normalization( + compiler, + tempdir, + false, + preprocessor_cache_mode, + ); } } @@ -564,7 +583,7 @@ fn test_cuda_compiles(compiler: &Compiler, tempdir: &Path) { assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); }); // By compiling another input source we verify that the pre-processor - // phase is correctly running and outputing text + // phase is correctly running and outputting text trace!("compile B"); sccache_command() .args(&compile_cuda_cmdline( @@ -694,7 +713,12 @@ fn test_clang_multicall(compiler: Compiler, tempdir: &Path) { .success(); } -fn test_clang_cache_whitespace_normalization(compiler: Compiler, tempdir: &Path, hit: bool) { +fn test_clang_cache_whitespace_normalization( + compiler: Compiler, + tempdir: &Path, + hit: bool, + preprocessor_cache_mode: bool, +) { let Compiler { name, exe, @@ -742,11 +766,18 @@ fn test_clang_cache_whitespace_normalization(compiler: Compiler, tempdir: &Path, .success(); println!("request stats (expecting cache hit)"); if hit { - get_stats(|info| { + get_stats(move |info| { assert_eq!(2, info.stats.compile_requests); assert_eq!(2, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); + if preprocessor_cache_mode { + // Preprocessor cache mode hashes the input file, so whitespace + // normalization does not work. + assert_eq!(0, info.stats.cache_hits.all()); + assert_eq!(2, info.stats.cache_misses.all()); + } else { + assert_eq!(1, info.stats.cache_hits.all()); + assert_eq!(1, info.stats.cache_misses.all()); + } }); } else { get_stats(|info| { @@ -816,10 +847,11 @@ fn find_cuda_compilers() -> Vec { // split up to run them individually. In the current form, it is hard to see // which sub test cases are executed, and if one fails, the remaining tests // are not run. -#[test] +#[test_case(true ; "with preprocessor cache")] +#[test_case(false ; "without preprocessor cache")] #[serial] #[cfg(any(unix, target_env = "msvc"))] -fn test_sccache_command() { +fn test_sccache_command(preprocessor_cache_mode: bool) { let _ = env_logger::try_init(); let tempdir = tempfile::Builder::new() .prefix("sccache_system_test") @@ -832,7 +864,7 @@ fn test_sccache_command() { // Ensure there's no existing sccache server running. stop_local_daemon(); // Create the configurations - let sccache_cfg = sccache_client_cfg(tempdir.path()); + let sccache_cfg = sccache_client_cfg(tempdir.path(), preprocessor_cache_mode); write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg); let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg"); // Start a server. @@ -842,17 +874,18 @@ fn test_sccache_command() { &sccache_cached_cfg_path, ); for compiler in compilers { - run_sccache_command_tests(compiler, tempdir.path()); + run_sccache_command_tests(compiler, tempdir.path(), preprocessor_cache_mode); zero_stats(); } stop_local_daemon(); } } -#[test] +#[test_case(true ; "with preprocessor cache")] +#[test_case(false ; "without preprocessor cache")] #[serial] #[cfg(any(unix, target_env = "msvc"))] -fn test_cuda_sccache_command() { +fn test_cuda_sccache_command(preprocessor_cache_mode: bool) { let _ = env_logger::try_init(); let tempdir = tempfile::Builder::new() .prefix("sccache_system_test") @@ -865,7 +898,7 @@ fn test_cuda_sccache_command() { // Ensure there's no existing sccache server running. stop_local_daemon(); // Create the configurations - let sccache_cfg = sccache_client_cfg(tempdir.path()); + let sccache_cfg = sccache_client_cfg(tempdir.path(), preprocessor_cache_mode); write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg); let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg"); // Start a server.