From 829f24f217a88fe68b31976974b5c57ad0599933 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 17:22:23 +0000 Subject: [PATCH 01/59] build(deps): bump rouille from 3.6.1 to 3.6.2 Bumps [rouille](https://github.com/tomaka/rouille) from 3.6.1 to 3.6.2. - [Release notes](https://github.com/tomaka/rouille/releases) - [Changelog](https://github.com/tomaka/rouille/blob/master/CHANGELOG.md) - [Commits](https://github.com/tomaka/rouille/compare/v3.6.1...v3.6.2) --- updated-dependencies: - dependency-name: rouille dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 7 +++---- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a123c1835..9395d4d15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2166,21 +2166,20 @@ dependencies = [ [[package]] name = "rouille" -version = "3.6.1" +version = "3.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f86e4c51a773f953f02bbab5fd049f004bfd384341d62da2a079aff812ab176" +checksum = "3716fbf57fc1084d7a706adf4e445298d123e4a44294c4e8213caf1b85fcc921" dependencies = [ "base64 0.13.1", "chrono", "filetime", "multipart", - "num_cpus", "percent-encoding", "rand", "serde", "serde_derive", "serde_json", - "sha1", + "sha1_smol", "threadpool", "time", "tiny_http", diff --git a/Cargo.toml b/Cargo.toml index 107e01f50..d6d1f04db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,7 +109,7 @@ zstd = "0.12" memmap2 = "0.9.0" nix = { version = "0.26.2", optional = true } object = "0.30" -rouille = { version = "3.5", optional = true, default-features = false, features = [ +rouille = { version = "3.6", optional = true, default-features = false, features = [ "ssl", ] } syslog = { version = "6", optional = true } From b46085bfc9396c34ae1f057006e3233fdbcc06df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:38:08 +0000 Subject: [PATCH 02/59] build(deps): bump flate2 from 1.0.26 to 1.0.28 Bumps [flate2](https://github.com/rust-lang/flate2-rs) from 1.0.26 to 1.0.28. - [Release notes](https://github.com/rust-lang/flate2-rs/releases) - [Commits](https://github.com/rust-lang/flate2-rs/compare/1.0.26...1.0.28) --- updated-dependencies: - dependency-name: flate2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9395d4d15..108e3426e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,9 +761,9 @@ checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide 0.7.1", From bef4ea255b0eab92a13e4814c27eb15f6e82609a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:38:24 +0000 Subject: [PATCH 03/59] build(deps): bump syslog from 6.0.1 to 6.1.0 Bumps [syslog](https://github.com/Geal/rust-syslog) from 6.0.1 to 6.1.0. - [Commits](https://github.com/Geal/rust-syslog/compare/6.0.1...6.1.0) --- updated-dependencies: - dependency-name: syslog dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 108e3426e..09845821e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2720,9 +2720,9 @@ dependencies = [ [[package]] name = "syslog" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978044cc68150ad5e40083c9f6a725e6fd02d7ba1bcf691ec2ff0d66c0b41acc" +checksum = "7434e95bcccce1215d30f4bf84fe8c00e8de1b9be4fb736d747ca53d36e7f96f" dependencies = [ "error-chain", "hostname", From 993998fed5082a1a4679835c16760f57f7cf4c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:38:34 +0000 Subject: [PATCH 04/59] build(deps): bump serde from 1.0.159 to 1.0.190 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.159 to 1.0.190. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.159...v1.0.190) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09845821e..0781873ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2464,18 +2464,18 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.159" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", From 41473c1c42d5d90f16f09c420d95eb1aec075f45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:38:42 +0000 Subject: [PATCH 05/59] build(deps): bump walkdir from 2.3.3 to 2.4.0 Bumps [walkdir](https://github.com/BurntSushi/walkdir) from 2.3.3 to 2.4.0. - [Commits](https://github.com/BurntSushi/walkdir/compare/2.3.3...2.4.0) --- updated-dependencies: - dependency-name: walkdir dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0781873ce..605a8551b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3313,9 +3313,9 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", From 1253ca875327cbee4d012c54ebb9865da4bce03e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:53:17 +0000 Subject: [PATCH 06/59] build(deps): bump once_cell from 1.17.1 to 1.18.0 Bumps [once_cell](https://github.com/matklad/once_cell) from 1.17.1 to 1.18.0. - [Changelog](https://github.com/matklad/once_cell/blob/master/CHANGELOG.md) - [Commits](https://github.com/matklad/once_cell/compare/v1.17.1...v1.18.0) --- updated-dependencies: - dependency-name: once_cell dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 605a8551b..1a232ac31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1634,9 +1634,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opendal" diff --git a/Cargo.toml b/Cargo.toml index d6d1f04db..2279041b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ log = "0.4" memchr = "2" num_cpus = "1.15" number_prefix = "0.4" -once_cell = "1.17" +once_cell = "1.18" opendal = { version = "0.40.0", optional = true } openssl = { version = "0.10.55", optional = true } rand = "0.8.4" From 0c3e7a4b88e54d96a6e25859c21b7ccbbe5e7ca4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 18:00:45 +0000 Subject: [PATCH 07/59] build(deps): bump assert_cmd from 2.0.10 to 2.0.11 Bumps [assert_cmd](https://github.com/assert-rs/assert_cmd) from 2.0.10 to 2.0.11. - [Changelog](https://github.com/assert-rs/assert_cmd/blob/master/CHANGELOG.md) - [Commits](https://github.com/assert-rs/assert_cmd/compare/v2.0.10...v2.0.11) --- updated-dependencies: - dependency-name: assert_cmd dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 14 ++++---------- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1a232ac31..8e8c4a1f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,12 +41,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstyle" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" - [[package]] name = "anstyle" version = "1.0.0" @@ -100,11 +94,11 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "assert_cmd" -version = "2.0.10" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b2340f55d9661d76793b2bfc2eb0e62689bd79d067a95707ea762afd5e9dd" +checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151" dependencies = [ - "anstyle 0.3.5", + "anstyle", "bstr", "doc-comment", "predicates", @@ -1861,7 +1855,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09963355b9f467184c04017ced4a2ba2d75cbcb4e7462690d388233253d4b1a9" dependencies = [ - "anstyle 1.0.0", + "anstyle", "difflib", "float-cmp", "itertools", diff --git a/Cargo.toml b/Cargo.toml index 2279041b8..173979a2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,7 +116,7 @@ syslog = { version = "6", optional = true } version-compare = { version = "0.1.1", optional = true } [dev-dependencies] -assert_cmd = "2.0.10" +assert_cmd = "2.0.11" cc = "1.0" chrono = "0.4.26" itertools = "0.10" From c60f1916734b4e15aee8f5438512c12a8312442e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 08:17:04 +0000 Subject: [PATCH 08/59] build(deps): bump num_cpus from 1.15.0 to 1.16.0 Bumps [num_cpus](https://github.com/seanmonstar/num_cpus) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/seanmonstar/num_cpus/releases) - [Changelog](https://github.com/seanmonstar/num_cpus/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/num_cpus/compare/v1.15.0...v1.16.0) --- updated-dependencies: - dependency-name: num_cpus dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 17 ++++------------- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e8c4a1f1..4b67ed056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -997,15 +997,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.1" @@ -1235,7 +1226,7 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi", "io-lifetimes", "rustix 0.37.7", "windows-sys 0.48.0", @@ -1593,11 +1584,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index 173979a2b..e49d28a86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ libc = "0.2.140" linked-hash-map = "0.5" log = "0.4" memchr = "2" -num_cpus = "1.15" +num_cpus = "1.16" number_prefix = "0.4" once_cell = "1.18" opendal = { version = "0.40.0", optional = true } From c7a4f1e591d086b4204661739287d76342eb38e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 16:55:52 +0100 Subject: [PATCH 09/59] build(deps): bump regex from 1.7.3 to 1.8.4 (#1794) Bumps [regex](https://github.com/rust-lang/regex) from 1.7.3 to 1.8.4. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.7.3...1.8.4) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sylvestre Ledru --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b67ed056..918e400fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] @@ -2017,9 +2017,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", @@ -2034,9 +2034,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "reqsign" diff --git a/Cargo.toml b/Cargo.toml index e49d28a86..98baa639c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ once_cell = "1.18" opendal = { version = "0.40.0", optional = true } openssl = { version = "0.10.55", optional = true } rand = "0.8.4" -regex = "1.7.3" +regex = "1.8.4" reqsign = { version = "0.14.1", optional = true } reqwest = { version = "0.11", features = [ "json", From b831172af097c2f9a523047ea4b06100462db5b0 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sat, 4 Nov 2023 21:22:14 +0100 Subject: [PATCH 10/59] update to libc 0.2.149 (#1933) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 98baa639c..005980b30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ hyper = { version = "0.14.25", optional = true, features = ["server"] } is-terminal = "0.4.7" jobserver = "0.1" jwt = { package = "jsonwebtoken", version = "8", optional = true } -libc = "0.2.140" +libc = "0.2.149" linked-hash-map = "0.5" log = "0.4" memchr = "2" From 7dc99ab3deb2755a6465cd8c1b7cb1eca8bcba8c Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 6 Nov 2023 11:44:23 +0100 Subject: [PATCH 11/59] Document how to set the direct access (#1936) @Alphare ok with you? --- docs/Local.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/Local.md b/docs/Local.md index d8fb381a6..5ccecf9f3 100644 --- a/docs/Local.md +++ b/docs/Local.md @@ -10,6 +10,11 @@ The local storage only supports a single sccache server at a time. Multiple conc This is inspired by [ccache's direct mode](https://ccache.dev/manual/3.7.9.html#_the_direct_mode) and works roughly the same. +It can be set with by setting the variable: +``` +SCCACHE_DIRECT=true +``` + In preprocessor cache mode, sccache caches the preprocessor step for C/C++ whenever possible. This can make the compilation a lot faster, since the preprocessor accounts for a non-negligible amount of time in the entire compile chain. In order to cache the preprocessor step sccache needs to remember, among other things, all files included by the given input file. To quote ccache's documentation: From c5d1ab4434c5b2389b2b6d4be9b0e3d272c585b1 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 6 Nov 2023 11:44:59 +0100 Subject: [PATCH 12/59] direct access: adjust the doc (#1935) --- docs/Configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index acfc410ed..dcc0a85bc 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -99,7 +99,7 @@ configuration variables * `SCCACHE_DIR` local on disk artifact cache directory * `SCCACHE_CACHE_SIZE` maximum size of the local on disk cache i.e. `2G` - default is 10G -* `SCCACHE_PREPROCESSOR_MODE` enable/disable preprocessor caching (see [the local doc](Local.md)) +* `SCCACHE_DIRECT` enable/disable preprocessor caching (see [the local doc](Local.md)) #### s3 compatible From 57024431eb6aa5cf13e7f4570b24652806919c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Mon, 6 Nov 2023 17:47:12 +0100 Subject: [PATCH 13/59] Allow using preprocessor mode from env without a disk config If we set `SCCACHE_DIRECT=true` but not `disk_dir` or `disk_sz`, we end up with `None` which sets `use_preprocessor_cache_mode` to `false`. But if we run: ``` ./target/debug/sccache --start-server ./target/debug/sccache --show-stats ``` We can see that it is uses the local disk by default. After this change, this works: `SCCACHE_DIRECT=true ./target/debug/sccache --start-server ./target/debug/sccache --show-stats` --- src/config.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/config.rs b/src/config.rs index c8614d31e..0d3823cce 100644 --- a/src/config.rs +++ b/src/config.rs @@ -710,13 +710,20 @@ fn config_from_env() -> Result { .and_then(|v| parse_size(&v)); let mut preprocessor_mode_config = PreprocessorCacheModeConfig::default(); - match env::var("SCCACHE_DIRECT").as_deref() { - Ok("on") | Ok("true") => preprocessor_mode_config.use_preprocessor_cache_mode = true, - Ok("off") | Ok("false") => preprocessor_mode_config.use_preprocessor_cache_mode = false, - _ => {} + let preprocessor_mode_overridden = match env::var("SCCACHE_DIRECT").as_deref() { + Ok("on") | Ok("true") => { + preprocessor_mode_config.use_preprocessor_cache_mode = true; + true + } + Ok("off") | Ok("false") => { + preprocessor_mode_config.use_preprocessor_cache_mode = false; + true + } + _ => false, }; - let disk = if disk_dir.is_some() || disk_sz.is_some() { + let any_overridden = disk_dir.is_some() || disk_sz.is_some() || preprocessor_mode_overridden; + let disk = if any_overridden { Some(DiskCacheConfig { dir: disk_dir.unwrap_or_else(default_disk_cache_dir), size: disk_sz.unwrap_or_else(default_disk_cache_size), From ce2599e4aed63039f3715b3b62c7cbe0e872c2c4 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 6 Nov 2023 14:27:48 +0100 Subject: [PATCH 14/59] Add a suggestion in case of error message --- src/commands.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands.rs b/src/commands.rs index 8805a157c..dc772488c 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -340,9 +340,9 @@ pub fn request_zero_stats(mut conn: ServerConnection) -> Result { /// Send a `GetStats` request to the server, and return the `ServerInfo` request if successful. pub fn request_stats(mut conn: ServerConnection) -> Result { debug!("request_stats"); - let response = conn - .request(Request::GetStats) - .context("Failed to send data to or receive data from server")?; + let response = conn.request(Request::GetStats).context( + "Failed to send data to or receive data from server. Mismatch of client/server versions?", + )?; if let Response::Stats(stats) = response { Ok(*stats) } else { From 8ed78c1fa271d64cbc6968ad3a8ccd97cc280e65 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 6 Nov 2023 14:58:08 +0100 Subject: [PATCH 15/59] add a check to fail quickly if SCCACHE_DIRECT is set but no directory info --- src/config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config.rs b/src/config.rs index c8614d31e..797899165 100644 --- a/src/config.rs +++ b/src/config.rs @@ -716,6 +716,12 @@ fn config_from_env() -> Result { _ => {} }; + if preprocessor_mode_config.use_preprocessor_cache_mode + && (disk_dir.is_none() && disk_sz.is_none()) + { + bail!("Direct access/mode has been set but no configuration about disk configuration"); + } + let disk = if disk_dir.is_some() || disk_sz.is_some() { Some(DiskCacheConfig { dir: disk_dir.unwrap_or_else(default_disk_cache_dir), From 7f5da74175b8649bd5b1043bae57fe623b8203c7 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 6 Nov 2023 15:00:19 +0100 Subject: [PATCH 16/59] --show-stats should show if we are using direct access/mode --- src/server.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/server.rs b/src/server.rs index 8237c3d50..239c96994 100644 --- a/src/server.rs +++ b/src/server.rs @@ -896,6 +896,10 @@ where async fn get_info(&self) -> Result { let stats = self.stats.lock().await.clone(); let cache_location = self.storage.location(); + let use_preprocessor_cache_mode = self + .storage + .preprocessor_cache_mode_config() + .use_preprocessor_cache_mode; let version = env!("CARGO_PKG_VERSION").to_string(); futures::try_join!(self.storage.current_size(), self.storage.max_size()).map( move |(cache_size, max_cache_size)| ServerInfo { @@ -903,6 +907,7 @@ where cache_location, cache_size, max_cache_size, + use_preprocessor_cache_mode, version, }, ) @@ -1442,6 +1447,7 @@ pub struct ServerInfo { pub cache_location: String, pub cache_size: Option, pub max_cache_size: Option, + pub use_preprocessor_cache_mode: bool, pub version: String, } @@ -1651,6 +1657,18 @@ impl ServerInfo { self.cache_location, name_width = name_width ); + if self.cache_location.starts_with("Local disk") { + println!( + "{: Date: Mon, 6 Nov 2023 15:00:39 +0100 Subject: [PATCH 17/59] Cosmetic: add some spaces to separate blocks --- tests/harness/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index f53890543..27351c279 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -154,6 +154,7 @@ pub fn sccache_client_cfg(tmpdir: &Path) -> sccache::config::FileConfig { server_startup_timeout_ms: None, } } + #[cfg(feature = "dist-server")] fn sccache_scheduler_cfg() -> sccache::config::scheduler::Config { sccache::config::scheduler::Config { @@ -164,6 +165,7 @@ fn sccache_scheduler_cfg() -> sccache::config::scheduler::Config { }, } } + #[cfg(feature = "dist-server")] fn sccache_server_cfg( tmpdir: &Path, From 1aa3e80701a580927d6cfbb6b0462d6a59d22d75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:43:36 +0000 Subject: [PATCH 18/59] build(deps): bump filetime from 0.2.21 to 0.2.22 Bumps [filetime](https://github.com/alexcrichton/filetime) from 0.2.21 to 0.2.22. - [Commits](https://github.com/alexcrichton/filetime/compare/0.2.21...0.2.22) --- updated-dependencies: - dependency-name: filetime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 918e400fa..b031863f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -737,13 +737,13 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "windows-sys 0.48.0", ] @@ -1746,7 +1746,7 @@ checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "windows-sys 0.42.0", ] @@ -2004,6 +2004,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -2011,7 +2020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] @@ -2744,7 +2753,7 @@ checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if 1.0.0", "fastrand", - "redox_syscall", + "redox_syscall 0.2.16", "rustix 0.36.16", "windows-sys 0.42.0", ] From 037c7575a8ef8ee22fc39c94bb2689ebf4450773 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:43:27 +0000 Subject: [PATCH 19/59] build(deps): bump blake3 from 1.3.3 to 1.5.0 Bumps [blake3](https://github.com/BLAKE3-team/BLAKE3) from 1.3.3 to 1.5.0. - [Release notes](https://github.com/BLAKE3-team/BLAKE3/releases) - [Commits](https://github.com/BLAKE3-team/BLAKE3/compare/1.3.3...1.5.0) --- updated-dependencies: - dependency-name: blake3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b031863f1..e47a9ab05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,9 +82,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "ascii" @@ -218,16 +218,15 @@ checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" [[package]] name = "blake3" -version = "1.3.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", - "arrayvec 0.7.2", + "arrayvec 0.7.4", "cc", "cfg-if 1.0.0", "constant_time_eq", - "digest", ] [[package]] @@ -403,9 +402,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" From 3ead089a36d70a0ed9cd698e5ba289f5223fb05b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:43:18 +0000 Subject: [PATCH 20/59] build(deps): bump hyper from 0.14.25 to 0.14.27 Bumps [hyper](https://github.com/hyperium/hyper) from 0.14.25 to 0.14.27. - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/v0.14.27/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v0.14.25...v0.14.27) --- updated-dependencies: - dependency-name: hyper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e47a9ab05..f31ca9365 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1079,9 +1079,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index 005980b30..c5c42c40b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ gzp = { version = "0.11.3", default-features = false, features = [ "deflate_rust", ] } http = "0.2" -hyper = { version = "0.14.25", optional = true, features = ["server"] } +hyper = { version = "0.14.27", optional = true, features = ["server"] } is-terminal = "0.4.7" jobserver = "0.1" jwt = { package = "jsonwebtoken", version = "8", optional = true } From 57b263ed0da427fb21ed23f39562211efa137e9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:42:51 +0000 Subject: [PATCH 21/59] build(deps): bump assert_cmd from 2.0.11 to 2.0.12 Bumps [assert_cmd](https://github.com/assert-rs/assert_cmd) from 2.0.11 to 2.0.12. - [Changelog](https://github.com/assert-rs/assert_cmd/blob/master/CHANGELOG.md) - [Commits](https://github.com/assert-rs/assert_cmd/compare/v2.0.11...v2.0.12) --- updated-dependencies: - dependency-name: assert_cmd dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f31ca9365..414aabd8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,9 +94,9 @@ checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] name = "assert_cmd" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" dependencies = [ "anstyle", "bstr", diff --git a/Cargo.toml b/Cargo.toml index c5c42c40b..7abeb1cf8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,7 +116,7 @@ syslog = { version = "6", optional = true } version-compare = { version = "0.1.1", optional = true } [dev-dependencies] -assert_cmd = "2.0.11" +assert_cmd = "2.0.12" cc = "1.0" chrono = "0.4.26" itertools = "0.10" From 16b2034500cddc9f49733f32b0d42f1aefbb8a4f Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 6 Nov 2023 15:12:18 +0100 Subject: [PATCH 22/59] add a pre-commit conf --- .pre-commit-config.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..f90466bed --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: +- repo: local + hooks: + - id: rust-linting + name: Rust linting + description: Run cargo fmt on files included in the commit. + entry: cargo +nightly fmt -- + pass_filenames: true + types: [file, rust] + language: system + - id: rust-clippy + name: Rust clippy + description: Run cargo clippy on files included in the commit. + entry: cargo +nightly clippy --workspace --all-targets --all-features -- + pass_filenames: false + types: [file, rust] + language: system From a2db9fb124d012b56f8faa8109e619dbe3f109ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:43:08 +0000 Subject: [PATCH 23/59] build(deps): bump byteorder from 1.4.3 to 1.5.0 Bumps [byteorder](https://github.com/BurntSushi/byteorder) from 1.4.3 to 1.5.0. - [Changelog](https://github.com/BurntSushi/byteorder/blob/master/CHANGELOG.md) - [Commits](https://github.com/BurntSushi/byteorder/compare/1.4.3...1.5.0) --- updated-dependencies: - dependency-name: byteorder dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 414aabd8b..ea5cc8eae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,9 +268,9 @@ checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" diff --git a/Cargo.toml b/Cargo.toml index 7abeb1cf8..437f376ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ async-trait = "0.1" base64 = "0.21" bincode = "1" blake3 = "1" -byteorder = "1.0" +byteorder = "1.5" bytes = "1" chrono = "0.4" clap = { version = "4.1.11", features = ["derive", "env", "wrap_help"] } From 1f04ad7cd29997a8ef75109ac2a3b553313e42a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:43:00 +0000 Subject: [PATCH 24/59] build(deps): bump toml from 0.7.6 to 0.8.6 Bumps [toml](https://github.com/toml-rs/toml) from 0.7.6 to 0.8.6. - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.7.6...toml-v0.8.6) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea5cc8eae..d7eab8b7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2500,9 +2500,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" dependencies = [ "serde", ] @@ -3015,9 +3015,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.6" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +checksum = "8ff9e3abce27ee2c9a37f9ad37238c1bdd4e789c84ba37df76aa4d528f5072cc" dependencies = [ "serde", "serde_spanned", @@ -3027,18 +3027,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ "indexmap 2.0.0", "serde", diff --git a/Cargo.toml b/Cargo.toml index 437f376ae..c15392416 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,7 @@ tokio = { version = "1", features = [ ] } tokio-serde = "0.8" tokio-util = { version = "0.7", features = ["codec", "io"] } -toml = "0.7" +toml = "0.8" tower = "0.4" trust-dns-resolver = { version = "0.22.0", features = [ "dnssec-ring", From 891e65a8b373acae14a3a2f7d849f1c9e989703e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:42:43 +0000 Subject: [PATCH 25/59] build(deps): bump directories from 5.0.0 to 5.0.1 Bumps [directories](https://github.com/soc/directories-rs) from 5.0.0 to 5.0.1. - [Commits](https://github.com/soc/directories-rs/commits) --- updated-dependencies: - dependency-name: directories dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 17 ++++++++++++----- Cargo.toml | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7eab8b7f..9b47001df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -541,22 +541,23 @@ dependencies = [ [[package]] name = "directories" -version = "5.0.0" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74be3be809c18e089de43bdc504652bb2bc473fca8756131f8689db8cf079ba9" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" dependencies = [ "dirs-sys", ] [[package]] name = "dirs-sys" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04414300db88f70d74c5ff54e50f9e1d1737d9a5b90f53fcf2e95ca2a9ab554b" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", + "option-ext", "redox_users", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1711,6 +1712,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-multimap" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index c15392416..6e96ae25b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ byteorder = "1.5" bytes = "1" chrono = "0.4" clap = { version = "4.1.11", features = ["derive", "env", "wrap_help"] } -directories = "5.0.0" +directories = "5.0.1" encoding = "0.2" env_logger = "0.10" filetime = "0.2" From 95138785383021aa8ce50b456ebfbef1277840ee Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 7 Nov 2023 11:41:50 +0100 Subject: [PATCH 26/59] When fails to connect, show the error Will help with issue #1938 --- src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index e8780bd16..51a6a1fe3 100644 --- a/src/client.rs +++ b/src/client.rs @@ -81,9 +81,9 @@ pub fn connect_with_retry(port: u16) -> io::Result { // us once it starts the server instead of us polling. match retry(Fixed::from_millis(500).take(10), || connect_to_server(port)) { Ok(conn) => Ok(conn), - _ => Err(io::Error::new( + Err(e) => Err(io::Error::new( io::ErrorKind::TimedOut, - "Connection to server timed out", + format!("Connection to server timed out: {:?}", e), )), } } From 1d71d5d810c161ff85217c71608d1d2262c35afb Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 7 Nov 2023 14:10:05 +0100 Subject: [PATCH 27/59] prepare version 0.7.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b47001df..bf48437fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "sccache" -version = "0.7.0" +version = "0.7.1" dependencies = [ "anyhow", "ar", diff --git a/Cargo.toml b/Cargo.toml index 6e96ae25b..73690df4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" name = "sccache" rust-version = "1.67.1" -version = "0.7.0" +version = "0.7.1" categories = ["command-line-utilities", "development-tools::build-utils"] description = "Sccache is a ccache-like tool. It is used as a compiler wrapper and avoids compilation when possible, storing a cache in a remote storage using various cloud storage." From 78ce03ceeda9289f63f4cd6b5f8ab3bd96dbdf23 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 14 Nov 2023 10:44:55 +0900 Subject: [PATCH 28/59] Fix off-by-one in line number replacement code Going through the current code fails with: ``` thread 'tokio-runtime-worker' panicked at src/compiler/c.rs:687:37: source slice length (3) does not match destination slice length (2) note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace ``` --- src/compiler/c.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/c.rs b/src/compiler/c.rs index 424105d5e..0e0b1da0c 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -684,7 +684,7 @@ fn process_preprocessor_line( // Replace the line number with the usual one. digest.update(&bytes[hash_start..start]); start += 1; - bytes[start..start + 2].copy_from_slice(b"# 1"); + bytes[start..=start + 2].copy_from_slice(b"# 1"); hash_start = start; slice = &bytes[start..]; } From 01bfa41f3a20a9f1154711a31c187d973d281dcb Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 14 Nov 2023 11:55:14 +0900 Subject: [PATCH 29/59] Fail early if the local daemon doesn't start properly during tests --- tests/harness/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 27351c279..1e828cc9a 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -43,7 +43,7 @@ const TC_CACHE_SIZE: u64 = 1024 * 1024 * 1024; // 1 gig pub fn start_local_daemon(cfg_path: &Path, cached_cfg_path: &Path) { // Don't run this with run() because on Windows `wait_with_output` // will hang because the internal server process is not detached. - let _ = sccache_command() + if !sccache_command() .arg("--start-server") // Uncomment following lines to debug locally. .env("SCCACHE_LOG", "debug") @@ -52,7 +52,10 @@ pub fn start_local_daemon(cfg_path: &Path, cached_cfg_path: &Path) { .env("SCCACHE_CACHED_CONF", cached_cfg_path) .status() .unwrap() - .success(); + .success() + { + panic!("Failed to start local daemon"); + } } pub fn stop_local_daemon() { From 725ae6fc30938ad88908c8f0ac3cd76c965fa639 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 14 Nov 2023 12:52:59 +0900 Subject: [PATCH 30/59] Fix location of the local daemon log in the test harness /tmp is a location that will usually not be user-writable on Windows. --- tests/harness/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index 1e828cc9a..c82f7abd0 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -47,7 +47,10 @@ pub fn start_local_daemon(cfg_path: &Path, cached_cfg_path: &Path) { .arg("--start-server") // Uncomment following lines to debug locally. .env("SCCACHE_LOG", "debug") - .env("SCCACHE_ERROR_LOG", "/tmp/sccache_local_daemon.txt") + .env( + "SCCACHE_ERROR_LOG", + env::temp_dir().join("sccache_local_daemon.txt"), + ) .env("SCCACHE_CONF", cfg_path) .env("SCCACHE_CACHED_CONF", cached_cfg_path) .status() From 71a528fe09845ea4409a723d276c3aaf962ad00a Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 14 Nov 2023 12:54:39 +0900 Subject: [PATCH 31/59] Comment out the logging setup in the test harness It was accidentally uncommented in 520a30b6. --- tests/harness/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index c82f7abd0..f33c6692e 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -46,11 +46,11 @@ pub fn start_local_daemon(cfg_path: &Path, cached_cfg_path: &Path) { if !sccache_command() .arg("--start-server") // Uncomment following lines to debug locally. - .env("SCCACHE_LOG", "debug") - .env( - "SCCACHE_ERROR_LOG", - env::temp_dir().join("sccache_local_daemon.txt"), - ) + // .env("SCCACHE_LOG", "debug") + // .env( + // "SCCACHE_ERROR_LOG", + // env::temp_dir().join("sccache_local_daemon.txt"), + // ) .env("SCCACHE_CONF", cfg_path) .env("SCCACHE_CACHED_CONF", cached_cfg_path) .status() From 012424de58ce72f79e0b67bf667b48f4e3ffc9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 7 Nov 2023 10:42:46 +0100 Subject: [PATCH 32/59] Improve context for open error when opening the input file We now get the path that failed to open. --- src/compiler/preprocessor_cache.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compiler/preprocessor_cache.rs b/src/compiler/preprocessor_cache.rs index d7adaacd0..3d263348b 100644 --- a/src/compiler/preprocessor_cache.rs +++ b/src/compiler/preprocessor_cache.rs @@ -416,7 +416,8 @@ pub fn preprocessor_cache_entry_hash_key( let mut buf = vec![]; encode_path(&mut buf, input_file)?; m.update(&buf); - let reader = std::fs::File::open(input_file).context("while hashing the input file")?; + let reader = std::fs::File::open(input_file) + .with_context(|| format!("while hashing the input file '{}'", input_file.display()))?; let digest = if config.ignore_time_macros { Digest::reader_sync(reader)? From 9ad19a226671e083eadd00e213b7751463b16bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 7 Nov 2023 10:59:06 +0100 Subject: [PATCH 33/59] Add dummy input file to all tests that call the preprocessor This fixes a ENOENT for when the preprocessor cache tries to read it. --- src/compiler/compiler.rs | 16 ++++++++++++++++ src/test/tests.rs | 2 ++ 2 files changed, 18 insertions(+) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index bd33f1c85..f08adb033 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -1677,6 +1677,8 @@ LLVM version: 6.0", let pool = runtime.handle(); let arguments = ovec!["-c", "foo.c", "-o", "foo.o"]; let cwd = f.tempdir.path(); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let results: Vec<_> = [11, 12] .iter() @@ -1754,6 +1756,8 @@ LLVM version: 6.0", &pool, Default::default(), ); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let storage = Arc::new(storage); // Pretend to be GCC. next_command( @@ -1873,6 +1877,8 @@ LLVM version: 6.0", &pool, Default::default(), ); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let storage = Arc::new(storage); // Pretend to be GCC. next_command( @@ -1984,6 +1990,8 @@ LLVM version: 6.0", let pool = runtime.handle().clone(); let storage = MockStorage::new(None); let storage: Arc = Arc::new(storage); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); // Pretend to be GCC. next_command( &creator, @@ -2064,6 +2072,8 @@ LLVM version: 6.0", let f = TestFixture::new(); let runtime = Runtime::new().unwrap(); let pool = runtime.handle().clone(); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); // Make our storage wait 2ms for each get/put operation. let storage_delay = Duration::from_millis(2); let storage = MockStorage::new(Some(storage_delay)); @@ -2149,6 +2159,8 @@ LLVM version: 6.0", Default::default(), ); let storage = Arc::new(storage); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); // Pretend to be GCC. next_command( &creator, @@ -2271,6 +2283,8 @@ LLVM version: 6.0", // Pretend to be GCC. Also inject a fake object file that the subsequent // preprocessor failure should remove. let obj = f.tempdir.path().join("foo.o"); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let o = obj.clone(); next_command_calls(&creator, move |_| { let mut f = File::create(&o)?; @@ -2346,6 +2360,8 @@ LLVM version: 6.0", test_dist::ErrorSubmitToolchainClient::new(), test_dist::ErrorRunJobClient::new(), ]; + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); let storage = DiskCache::new( f.tempdir.path().join("cache"), u64::MAX, diff --git a/src/test/tests.rs b/src/test/tests.rs index 39a1317d4..54ad27f62 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -229,6 +229,8 @@ fn test_server_compile() { const STDOUT: &[u8] = b"some stdout"; const STDERR: &[u8] = b"some stderr"; let conn = connect_to_server(port).unwrap(); + // Write a dummy input file so the preprocessor cache mode can work + std::fs::write(f.tempdir.path().join("file.c"), "whatever").unwrap(); { let mut c = server_creator.lock().unwrap(); // The server will check the compiler. Pretend it's GCC. From a39b53bbd9bf35ae528b438d5a2a9578d5670fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 7 Nov 2023 11:29:50 +0100 Subject: [PATCH 34/59] Parametrize relevant tests with preprocessor cache Some compile tests should be run with and without preprocessor cache active. We introduce a new dev dependency that allows us to parametrize tests easily, with no duplication. --- Cargo.lock | 36 +++++++++++++++++++ Cargo.toml | 34 +++--------------- src/compiler/compiler.rs | 74 ++++++++++++++++++++++++++-------------- src/compiler/rust.rs | 55 ++++++++++++++++++----------- src/config.rs | 3 +- src/test/mock_storage.rs | 12 +++++-- tests/dist.rs | 2 +- tests/harness/mod.rs | 9 ++++- tests/system.rs | 53 ++++++++++++++++++++-------- 9 files changed, 184 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf48437fc..5419edfc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2390,6 +2390,7 @@ dependencies = [ "tar", "temp-env", "tempfile", + "test-case", "thirtyfour_sync", "tokio", "tokio-serde", @@ -2789,6 +2790,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" +[[package]] +name = "test-case" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8f1e820b7f1d95a0cdbf97a5df9de10e1be731983ab943e56703ac1b8e9d425" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54c25e2cb8f5fcd7318157634e8838aa6f7e4715c96637f969fabaccd1ef5462" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "test-case-macros" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37cfd7bbc88a0104e304229fba519bdc45501a30b760fb72240342f1289ad257" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.38", + "test-case-core", +] + [[package]] name = "thirtyfour" version = "0.27.3" diff --git a/Cargo.toml b/Cargo.toml index 73690df4d..3ae324588 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,7 @@ itertools = "0.10" predicates = "=3.0.3" serial_test = "2.0" temp-env = "0.3.4" +test-case = "3.2.1" thirtyfour_sync = "0.27" [target.'cfg(unix)'.dependencies] @@ -137,16 +138,7 @@ features = ["fileapi", "handleapi", "stringapiset", "winnls"] version = "0.3" [features] -all = [ - "dist-client", - "redis", - "s3", - "memcached", - "gcs", - "azure", - "gha", - "webdav", -] +all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure", "gha", "webdav"] azure = ["opendal", "reqsign"] default = ["all"] gcs = ["opendal", "reqsign", "url", "reqwest/blocking", "trust-dns-resolver"] @@ -163,27 +155,9 @@ vendored-openssl = ["openssl?/vendored", "opendal?/native-tls-vendored"] # Enable features that require unstable features of Nightly Rust. unstable = [] # Enables distributed support in the sccache client -dist-client = [ - "flate2", - "hyper", - "reqwest", - "url", - "sha2", - "trust-dns-resolver", -] +dist-client = ["flate2", "hyper", "reqwest", "url", "sha2", "trust-dns-resolver"] # Enables the sccache-dist binary -dist-server = [ - "jwt", - "flate2", - "libmount", - "nix", - "openssl", - "reqwest", - "rouille", - "syslog", - "trust-dns-resolver", - "version-compare", -] +dist-server = ["jwt", "flate2", "libmount", "nix", "openssl", "reqwest", "rouille", "syslog", "trust-dns-resolver", "version-compare"] # Enables dist tests with external requirements dist-tests = ["dist-client", "dist-server"] diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index f08adb033..a5768be93 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -1381,7 +1381,7 @@ where mod test { use super::*; use crate::cache::disk::DiskCache; - use crate::cache::CacheRead; + use crate::cache::{CacheRead, PreprocessorCacheModeConfig}; use crate::mock_command::*; use crate::test::mock_storage::MockStorage; use crate::test::utils::*; @@ -1390,6 +1390,7 @@ mod test { use std::sync::Arc; use std::time::Duration; use std::u64; + use test_case::test_case; use tokio::runtime::Runtime; #[test] @@ -1669,8 +1670,9 @@ LLVM version: 6.0", .is_err()); } - #[test] - fn test_compiler_version_affects_hash() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_version_affects_hash(preprocessor_cache_mode: bool) { let f = TestFixture::new(); let creator = new_creator(); let runtime = single_threaded_runtime(); @@ -1713,7 +1715,7 @@ LLVM version: 6.0", false, pool, false, - Arc::new(MockStorage::new(None)), + Arc::new(MockStorage::new(None, preprocessor_cache_mode)), CacheControl::Default, ) .wait() @@ -1743,8 +1745,9 @@ LLVM version: 6.0", assert_eq!(CompilerKind::C(CCompilerKind::Gcc), c.kind()); } - #[test] - fn test_compiler_get_cached_or_compile() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_get_cached_or_compile(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -1754,7 +1757,10 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); // Write a dummy input file so the preprocessor cache mode can work std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); @@ -1863,9 +1869,10 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] #[cfg(feature = "dist-client")] - fn test_compiler_get_cached_or_compile_dist() { + fn test_compiler_get_cached_or_compile_dist(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -1875,7 +1882,10 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); // Write a dummy input file so the preprocessor cache mode can work std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); @@ -1979,16 +1989,17 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] /// Test that a cache read that results in an error is treated as a cache /// miss. - fn test_compiler_get_cached_or_compile_cache_error() { + fn test_compiler_get_cached_or_compile_cache_error(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); let runtime = Runtime::new().unwrap(); let pool = runtime.handle().clone(); - let storage = MockStorage::new(None); + let storage = MockStorage::new(None, preprocessor_cache_mode); let storage: Arc = Arc::new(storage); // Write a dummy input file so the preprocessor cache mode can work std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); @@ -2064,9 +2075,10 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] /// Test that cache read timing is recorded. - fn test_compiler_get_cached_or_compile_cache_get_timing() { + fn test_compiler_get_cached_or_compile_cache_get_timing(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -2076,7 +2088,7 @@ LLVM version: 6.0", std::fs::write(f.tempdir.path().join("foo.c"), "whatever").unwrap(); // Make our storage wait 2ms for each get/put operation. let storage_delay = Duration::from_millis(2); - let storage = MockStorage::new(Some(storage_delay)); + let storage = MockStorage::new(Some(storage_delay), preprocessor_cache_mode); let storage: Arc = Arc::new(storage); // Pretend to be GCC. next_command( @@ -2145,8 +2157,9 @@ LLVM version: 6.0", } } - #[test] - fn test_compiler_get_cached_or_compile_force_recache() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_get_cached_or_compile_force_recache(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -2156,7 +2169,10 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); let storage = Arc::new(storage); // Write a dummy input file so the preprocessor cache mode can work @@ -2266,8 +2282,9 @@ LLVM version: 6.0", assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); } - #[test] - fn test_compiler_get_cached_or_compile_preprocessor_error() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_compiler_get_cached_or_compile_preprocessor_error(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -2277,7 +2294,10 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); let storage = Arc::new(storage); // Pretend to be GCC. Also inject a fake object file that the subsequent @@ -2346,9 +2366,10 @@ LLVM version: 6.0", assert!(fs::metadata(&obj).is_err()); } - #[test] + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] #[cfg(feature = "dist-client")] - fn test_compiler_get_cached_or_compile_dist_error() { + fn test_compiler_get_cached_or_compile_dist_error(preprocessor_cache_mode: bool) { drop(env_logger::try_init()); let creator = new_creator(); let f = TestFixture::new(); @@ -2366,7 +2387,10 @@ LLVM version: 6.0", f.tempdir.path().join("cache"), u64::MAX, &pool, - Default::default(), + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ); let storage = Arc::new(storage); // Pretend to be GCC. diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index d97113657..dca3ddf9e 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -2435,6 +2435,7 @@ mod test { use std::ffi::OsStr; use std::io::{self, Write}; use std::sync::{Arc, Mutex}; + use test_case::test_case; fn _parse_arguments(arguments: &[String]) -> CompilerArguments { let arguments = arguments.iter().map(OsString::from).collect::>(); @@ -3076,8 +3077,9 @@ proc_macro false ); } - #[test] - fn test_generate_hash_key() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_generate_hash_key(preprocessor_cache_mode: bool) { use ar::{Builder, Header}; drop(env_logger::try_init()); let f = TestFixture::new(); @@ -3160,7 +3162,7 @@ proc_macro false false, &pool, false, - Arc::new(MockStorage::new(None)), + Arc::new(MockStorage::new(None, preprocessor_cache_mode)), CacheControl::Default, ) .wait() @@ -3205,6 +3207,7 @@ proc_macro false args: &[&'static str], env_vars: &[(OsString, OsString)], pre_func: F, + preprocessor_cache_mode: bool, ) -> String where F: Fn(&Path) -> Result<()>, @@ -3251,7 +3254,7 @@ proc_macro false false, &pool, false, - Arc::new(MockStorage::new(None)), + Arc::new(MockStorage::new(None, preprocessor_cache_mode)), CacheControl::Default, ) .wait() @@ -3264,8 +3267,9 @@ proc_macro false Ok(()) } - #[test] - fn test_equal_hashes_externs() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_externs(preprocessor_cache_mode: bool) { // Put some content in the extern rlibs so we can verify that the content hashes are // used in the right order. fn mk_files(tempdir: &Path) -> Result<()> { @@ -3293,7 +3297,8 @@ proc_macro false "b=b.rlib" ], &[], - mk_files + mk_files, + preprocessor_cache_mode, ), hash_key( &f, @@ -3313,13 +3318,15 @@ proc_macro false "lib" ], &[], - mk_files + mk_files, + preprocessor_cache_mode, ) ); } - #[test] - fn test_equal_hashes_link_paths() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_link_paths(preprocessor_cache_mode: bool) { let f = TestFixture::new(); assert_eq!( hash_key( @@ -3340,7 +3347,8 @@ proc_macro false "y=y" ], &[], - nothing + nothing, + preprocessor_cache_mode, ), hash_key( &f, @@ -3360,13 +3368,15 @@ proc_macro false "lib" ], &[], - nothing + nothing, + preprocessor_cache_mode, ) ); } - #[test] - fn test_equal_hashes_ignored_args() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_ignored_args(preprocessor_cache_mode: bool) { let f = TestFixture::new(); assert_eq!( hash_key( @@ -3389,7 +3399,8 @@ proc_macro false "y=y" ], &[], - nothing + nothing, + preprocessor_cache_mode, ), hash_key( &f, @@ -3411,13 +3422,15 @@ proc_macro false "lib" ], &[], - nothing + nothing, + preprocessor_cache_mode, ) ); } - #[test] - fn test_equal_hashes_cfg_features() { + #[test_case(true ; "with preprocessor cache")] + #[test_case(false ; "without preprocessor cache")] + fn test_equal_hashes_cfg_features(preprocessor_cache_mode: bool) { let f = TestFixture::new(); assert_eq!( hash_key( @@ -3438,7 +3451,8 @@ proc_macro false "feature=b" ], &[], - nothing + nothing, + preprocessor_cache_mode, ), hash_key( &f, @@ -3458,7 +3472,8 @@ proc_macro false "lib" ], &[], - nothing + nothing, + preprocessor_cache_mode, ) ); } diff --git a/src/config.rs b/src/config.rs index 0d3823cce..1e7bca567 100644 --- a/src/config.rs +++ b/src/config.rs @@ -33,7 +33,8 @@ use std::result::Result as StdResult; use std::str::FromStr; use std::sync::Mutex; -use crate::{cache::PreprocessorCacheModeConfig, errors::*}; +pub use crate::cache::PreprocessorCacheModeConfig; +use crate::errors::*; static CACHED_CONFIG_PATH: Lazy = Lazy::new(CachedConfig::file_config_path); static CACHED_CONFIG: Lazy>> = Lazy::new(|| Mutex::new(None)); diff --git a/src/test/mock_storage.rs b/src/test/mock_storage.rs index 5cfa391d7..00a6aa7c8 100644 --- a/src/test/mock_storage.rs +++ b/src/test/mock_storage.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::cache::{Cache, CacheWrite, Storage}; +use crate::cache::{Cache, CacheWrite, PreprocessorCacheModeConfig, Storage}; use crate::errors::*; use async_trait::async_trait; use futures::channel::mpsc; @@ -26,16 +26,18 @@ pub struct MockStorage { rx: Arc>>>, tx: mpsc::UnboundedSender>, delay: Option, + preprocessor_cache_mode: bool, } impl MockStorage { /// Create a new `MockStorage`. if `delay` is `Some`, wait for that amount of time before returning from operations. - pub(crate) fn new(delay: Option) -> MockStorage { + pub(crate) fn new(delay: Option, preprocessor_cache_mode: bool) -> MockStorage { let (tx, rx) = mpsc::unbounded(); Self { tx, rx: Arc::new(Mutex::new(rx)), delay, + preprocessor_cache_mode, } } @@ -72,4 +74,10 @@ impl Storage for MockStorage { async fn max_size(&self) -> Result> { Ok(None) } + fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { + PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: self.preprocessor_cache_mode, + ..Default::default() + } + } } diff --git a/tests/dist.rs b/tests/dist.rs index 4885e0451..f9c0ec883 100644 --- a/tests/dist.rs +++ b/tests/dist.rs @@ -52,7 +52,7 @@ pub fn dist_test_sccache_client_cfg( tmpdir: &Path, scheduler_url: HTTPUrl, ) -> sccache::config::FileConfig { - let mut sccache_cfg = harness::sccache_client_cfg(tmpdir); + let mut sccache_cfg = harness::sccache_client_cfg(tmpdir, false); sccache_cfg.cache.disk.as_mut().unwrap().size = 0; sccache_cfg.dist.scheduler_url = Some(scheduler_url); sccache_cfg diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index f33c6692e..b46fc033d 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -128,7 +128,10 @@ pub fn sccache_dist_path() -> PathBuf { assert_cmd::cargo::cargo_bin("sccache-dist") } -pub fn sccache_client_cfg(tmpdir: &Path) -> sccache::config::FileConfig { +pub fn sccache_client_cfg( + tmpdir: &Path, + preprocessor_cache_mode: bool, +) -> sccache::config::FileConfig { let cache_relpath = "client-cache"; let dist_cache_relpath = "client-dist-cache"; fs::create_dir(tmpdir.join(cache_relpath)).unwrap(); @@ -136,6 +139,10 @@ pub fn sccache_client_cfg(tmpdir: &Path) -> sccache::config::FileConfig { let disk_cache = sccache::config::DiskCacheConfig { dir: tmpdir.join(cache_relpath), + preprocessor_cache_mode: sccache::config::PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, ..Default::default() }; sccache::config::FileConfig { diff --git a/tests/system.rs b/tests/system.rs index abf890fb0..b20aa72b5 100644 --- a/tests/system.rs +++ b/tests/system.rs @@ -38,6 +38,7 @@ use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::str; +use test_case::test_case; use which::{which, which_in}; mod harness; @@ -450,7 +451,7 @@ fn test_compile_with_define(compiler: Compiler, tempdir: &Path) { .stderr(predicates::str::contains("warning:").from_utf8().not()); } -fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path) { +fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path, preprocessor_cache_mode: bool) { if compiler.name != "clang++" { test_basic_compile(compiler.clone(), tempdir); } @@ -495,9 +496,19 @@ fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path) { version_output ), }; - test_clang_cache_whitespace_normalization(compiler, tempdir, !is_appleclang && major >= 14); + test_clang_cache_whitespace_normalization( + compiler, + tempdir, + !is_appleclang && major >= 14, + preprocessor_cache_mode, + ); } else { - test_clang_cache_whitespace_normalization(compiler, tempdir, false); + test_clang_cache_whitespace_normalization( + compiler, + tempdir, + false, + preprocessor_cache_mode, + ); } } @@ -694,7 +705,12 @@ fn test_clang_multicall(compiler: Compiler, tempdir: &Path) { .success(); } -fn test_clang_cache_whitespace_normalization(compiler: Compiler, tempdir: &Path, hit: bool) { +fn test_clang_cache_whitespace_normalization( + compiler: Compiler, + tempdir: &Path, + hit: bool, + preprocessor_cache_mode: bool, +) { let Compiler { name, exe, @@ -742,11 +758,18 @@ fn test_clang_cache_whitespace_normalization(compiler: Compiler, tempdir: &Path, .success(); println!("request stats (expecting cache hit)"); if hit { - get_stats(|info| { + get_stats(move |info| { assert_eq!(2, info.stats.compile_requests); assert_eq!(2, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); + if preprocessor_cache_mode { + // Preprocessor cache mode hashes the input file, so whitespace + // normalization does not work. + assert_eq!(0, info.stats.cache_hits.all()); + assert_eq!(2, info.stats.cache_misses.all()); + } else { + assert_eq!(1, info.stats.cache_hits.all()); + assert_eq!(1, info.stats.cache_misses.all()); + } }); } else { get_stats(|info| { @@ -816,10 +839,11 @@ fn find_cuda_compilers() -> Vec { // split up to run them individually. In the current form, it is hard to see // which sub test cases are executed, and if one fails, the remaining tests // are not run. -#[test] +#[test_case(true ; "with preprocessor cache")] +#[test_case(false ; "without preprocessor cache")] #[serial] #[cfg(any(unix, target_env = "msvc"))] -fn test_sccache_command() { +fn test_sccache_command(preprocessor_cache_mode: bool) { let _ = env_logger::try_init(); let tempdir = tempfile::Builder::new() .prefix("sccache_system_test") @@ -832,7 +856,7 @@ fn test_sccache_command() { // Ensure there's no existing sccache server running. stop_local_daemon(); // Create the configurations - let sccache_cfg = sccache_client_cfg(tempdir.path()); + let sccache_cfg = sccache_client_cfg(tempdir.path(), preprocessor_cache_mode); write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg); let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg"); // Start a server. @@ -842,17 +866,18 @@ fn test_sccache_command() { &sccache_cached_cfg_path, ); for compiler in compilers { - run_sccache_command_tests(compiler, tempdir.path()); + run_sccache_command_tests(compiler, tempdir.path(), preprocessor_cache_mode); zero_stats(); } stop_local_daemon(); } } -#[test] +#[test_case(true ; "with preprocessor cache")] +#[test_case(false ; "without preprocessor cache")] #[serial] #[cfg(any(unix, target_env = "msvc"))] -fn test_cuda_sccache_command() { +fn test_cuda_sccache_command(preprocessor_cache_mode: bool) { let _ = env_logger::try_init(); let tempdir = tempfile::Builder::new() .prefix("sccache_system_test") @@ -865,7 +890,7 @@ fn test_cuda_sccache_command() { // Ensure there's no existing sccache server running. stop_local_daemon(); // Create the configurations - let sccache_cfg = sccache_client_cfg(tempdir.path()); + let sccache_cfg = sccache_client_cfg(tempdir.path(), preprocessor_cache_mode); write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg); let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg"); // Start a server. From 9d9ea15f90e250a79b64be394a1486e6c8806208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 7 Nov 2023 11:58:41 +0100 Subject: [PATCH 35/59] Appease the TOML formatter --- Cargo.toml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3ae324588..0e06755b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,7 +138,16 @@ features = ["fileapi", "handleapi", "stringapiset", "winnls"] version = "0.3" [features] -all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure", "gha", "webdav"] +all = [ + "dist-client", + "redis", + "s3", + "memcached", + "gcs", + "azure", + "gha", + "webdav", +] azure = ["opendal", "reqsign"] default = ["all"] gcs = ["opendal", "reqsign", "url", "reqwest/blocking", "trust-dns-resolver"] @@ -155,9 +164,27 @@ vendored-openssl = ["openssl?/vendored", "opendal?/native-tls-vendored"] # Enable features that require unstable features of Nightly Rust. unstable = [] # Enables distributed support in the sccache client -dist-client = ["flate2", "hyper", "reqwest", "url", "sha2", "trust-dns-resolver"] +dist-client = [ + "flate2", + "hyper", + "reqwest", + "url", + "sha2", + "trust-dns-resolver", +] # Enables the sccache-dist binary -dist-server = ["jwt", "flate2", "libmount", "nix", "openssl", "reqwest", "rouille", "syslog", "trust-dns-resolver", "version-compare"] +dist-server = [ + "jwt", + "flate2", + "libmount", + "nix", + "openssl", + "reqwest", + "rouille", + "syslog", + "trust-dns-resolver", + "version-compare", +] # Enables dist tests with external requirements dist-tests = ["dist-client", "dist-server"] From b3e75d0861a73b71a1d9452a78328c53dc8fb3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 7 Nov 2023 14:06:03 +0100 Subject: [PATCH 36/59] Fix input file times in end-to-end tests Explanations inline. --- Cargo.toml | 1 + tests/system.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 0e06755b2..e140910f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,7 @@ version-compare = { version = "0.1.1", optional = true } assert_cmd = "2.0.12" cc = "1.0" chrono = "0.4.26" +filetime = "0.2" itertools = "0.10" predicates = "=3.0.3" serial_test = "2.0" diff --git a/tests/system.rs b/tests/system.rs index b20aa72b5..8feab0fa6 100644 --- a/tests/system.rs +++ b/tests/system.rs @@ -38,6 +38,7 @@ use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; use std::str; +use std::time::{Duration, SystemTime}; use test_case::test_case; use which::{which, which_in}; @@ -152,6 +153,13 @@ fn copy_to_tempdir(inputs: &[&str], tempdir: &Path) { let source_file = tempdir.join(f); trace!("fs::copy({:?}, {:?})", original_source_file, source_file); fs::copy(&original_source_file, &source_file).unwrap(); + // Preprocessor cache will not cache files that are too recent. + // Certain OS/FS combinations have a slow resolution (up to 2s for NFS), + // leading to flaky tests. + // We set the times for the new file to 10 seconds ago, to be safe. + let new_time = + filetime::FileTime::from_system_time(SystemTime::now() - Duration::from_secs(10)); + filetime::set_file_times(source_file, new_time, new_time).unwrap(); } } From e63185b6d434aac581d5da1037f51786e60df2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Tue, 7 Nov 2023 10:10:27 +0100 Subject: [PATCH 37/59] Activate preprocessor cache mode by default Preprocessor cache mode (or "direct mode" in ccache speak), offers a measurable improvement for C/C++ caching. A non-scientific test of compiling Firefox on a Ryzen 7950x with hot caches saw this mode changing the wall time from ~48s to ~13s. This is the overall time spent in `mach build` with the only work needed is to recompile all C++. As noted in the docs, this applies only to local storage. --- docs/Local.md | 2 +- src/cache/cache.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Local.md b/docs/Local.md index 5ccecf9f3..89525fbf0 100644 --- a/docs/Local.md +++ b/docs/Local.md @@ -30,7 +30,7 @@ Preprocessor cache mode will be disabled if any of the following holds: Configuration options and their default values: -- `use_preprocessor_cache_mode`: `false`. Whether to use preprocessor cache mode entirely. +- `use_preprocessor_cache_mode`: `true`. Whether to use preprocessor cache mode entirely. - `file_stat_matches`: `false`. If false, only compare header files by hashing their contents. If true, will use size + ctime + mtime to check whether a file has changed. See other flags below for more control over this behavior. - `use_ctime_for_stat`: `true`. If true, uses the ctime (file status change on UNIX, creation time on Windows) to check that a file has/hasn't changed. Can be useful to disable when backdating modification times in a controlled manner. diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 717cde09a..37a690950 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -366,7 +366,7 @@ pub trait Storage: Send + Sync { /// Return the config for preprocessor cache mode if applicable fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { - // Disabled by default, only enabled in local mode + // Enabled by default, only enabled in local mode PreprocessorCacheModeConfig::default() } /// Return the preprocessor cache entry for a given preprocessor key, @@ -421,7 +421,7 @@ pub struct PreprocessorCacheModeConfig { impl Default for PreprocessorCacheModeConfig { fn default() -> Self { Self { - use_preprocessor_cache_mode: false, + use_preprocessor_cache_mode: true, file_stat_matches: false, use_ctime_for_stat: true, ignore_time_macros: false, From 147aacf5f263326184a14d0a8f7a632ab0aa384c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:27:59 +0000 Subject: [PATCH 38/59] build(deps): bump semver from 1.0.17 to 1.0.20 Bumps [semver](https://github.com/dtolnay/semver) from 1.0.17 to 1.0.20. - [Release notes](https://github.com/dtolnay/semver/releases) - [Commits](https://github.com/dtolnay/semver/compare/1.0.17...1.0.20) --- updated-dependencies: - dependency-name: semver dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5419edfc9..bb8da29e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2459,9 +2459,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" From 033ea2a5f913f5c3341b5b93cf29eaa86fc6d361 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:27:41 +0000 Subject: [PATCH 39/59] build(deps): bump jobserver from 0.1.26 to 0.1.27 Bumps [jobserver](https://github.com/alexcrichton/jobserver-rs) from 0.1.26 to 0.1.27. - [Commits](https://github.com/alexcrichton/jobserver-rs/commits) --- updated-dependencies: - dependency-name: jobserver dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb8da29e9..fa076aca9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1249,9 +1249,9 @@ checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] From 17f20db702e106dbcc20cacefb038c59e9bb9f2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:27:05 +0000 Subject: [PATCH 40/59] build(deps): bump base64 from 0.21.3 to 0.21.5 Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.21.3 to 0.21.5. - [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md) - [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.21.3...v0.21.5) --- updated-dependencies: - dependency-name: base64 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa076aca9..b745187f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,9 +172,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.3" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64ct" @@ -1633,7 +1633,7 @@ dependencies = [ "async-compat", "async-trait", "backon", - "base64 0.21.3", + "base64 0.21.5", "bb8", "bytes", "chrono", @@ -2061,7 +2061,7 @@ checksum = "3228e570df74d69d3d3236a71371f1edd748a3e4eb728ea1f29d403bc10fc727" dependencies = [ "anyhow", "async-trait", - "base64 0.21.3", + "base64 0.21.5", "chrono", "form_urlencoded", "hex", @@ -2090,7 +2090,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.3", + "base64 0.21.5", "bytes", "encoding_rs", "futures-core", @@ -2336,7 +2336,7 @@ dependencies = [ "ar", "assert_cmd", "async-trait", - "base64 0.21.3", + "base64 0.21.5", "bincode", "blake3", "byteorder", From 8f371ef378eca076ee3edae02e7dd24982494f31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 08:58:40 +0000 Subject: [PATCH 41/59] build(deps): bump bytes from 1.4.0 to 1.5.0 Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b745187f2..558f5e1ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,9 +274,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cc" From 7358e8c27a7c21a2e1f70f9bfeab8f45f90d6e4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:27:23 +0000 Subject: [PATCH 42/59] build(deps): bump zstd from 0.12.4 to 0.13.0 Bumps [zstd](https://github.com/gyscos/zstd-rs) from 0.12.4 to 0.13.0. - [Release notes](https://github.com/gyscos/zstd-rs/releases) - [Commits](https://github.com/gyscos/zstd-rs/compare/v0.12.4...v0.13.0) --- updated-dependencies: - dependency-name: zstd dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 9 ++++----- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 558f5e1ac..1f6559e53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3765,20 +3765,19 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index e140910f7..0fdf6c4d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,7 +103,7 @@ walkdir = "2" # by default which pulls in an outdated failure version which = { version = "4", default-features = false } zip = { version = "0.6", default-features = false } -zstd = "0.12" +zstd = "0.13" # dist-server only memmap2 = "0.9.0" From 2f1bf9b128fa20cf22933613f766419d7130a4bd Mon Sep 17 00:00:00 2001 From: roblabla Date: Fri, 4 Aug 2023 22:46:34 +0200 Subject: [PATCH 43/59] Don't pass wd4668 to clang-cl --- src/compiler/msvc.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index e6474fd9d..248c9e195 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -885,9 +885,6 @@ where .args(&parsed_args.preprocessor_args) .args(&parsed_args.dependency_args) .args(&parsed_args.common_args) - // Windows SDK generates C4668 during preprocessing, but compiles fine. - // Read for more info: https://github.com/mozilla/sccache/issues/1725 - .arg("/wd4668") .env_clear() .envs(env_vars.iter().map(|(k, v)| (k, v))) .current_dir(cwd); @@ -902,6 +899,9 @@ where cmd.arg("/sourceDependencies"); cmd.arg(depfile); } + // Windows SDK generates C4668 during preprocessing, but compiles fine. + // Read for more info: https://github.com/mozilla/sccache/issues/1725 + cmd.arg("/wd4668"); } if rewrite_includes_only && is_clang { From 0aeceae3be8906ce79070c8dfecef3c4a3d308e8 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 14 Nov 2023 13:22:50 +0100 Subject: [PATCH 44/59] prepare version 0.7.2 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f6559e53..3b9c018b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "sccache" -version = "0.7.1" +version = "0.7.2" dependencies = [ "anyhow", "ar", diff --git a/Cargo.toml b/Cargo.toml index 0fdf6c4d3..94e2b799d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" name = "sccache" rust-version = "1.67.1" -version = "0.7.1" +version = "0.7.2" categories = ["command-line-utilities", "development-tools::build-utils"] description = "Sccache is a ccache-like tool. It is used as a compiler wrapper and avoids compilation when possible, storing a cache in a remote storage using various cloud storage." From f6fd5ddd2ad8370cada6b106fb254de099f96da5 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 14 Nov 2023 21:26:43 +0100 Subject: [PATCH 45/59] Fix some typos --- README.md | 2 +- docs/ResponseFiles.md | 4 ++-- snap/snapcraft.yaml | 2 +- src/compiler/msvc.rs | 4 ++-- src/compiler/nvhpc.rs | 2 +- src/config.rs | 2 +- tests/system.rs | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6e5bd6a1d..cf5cd660f 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ sccache includes support for caching the compilation of C/C++ code, [Rust](docs/ sccache also provides [icecream](https://github.com/icecc/icecream)-style distributed compilation (automatic packaging of local toolchains) for all supported compilers (including Rust). The distributed compilation system includes several security features that icecream lacks such as authentication, transport layer encryption, and sandboxed compiler execution on build servers. See [the distributed quickstart](docs/DistributedQuickstart.md) guide for more information. -sccache is also available as a [GitHub Actions](https://github.com/marketplace/actions/sccache-action) to faciliate the deployment using GitHub Actions cache. +sccache is also available as a [GitHub Actions](https://github.com/marketplace/actions/sccache-action) to facilitate the deployment using GitHub Actions cache. --- diff --git a/docs/ResponseFiles.md b/docs/ResponseFiles.md index 9c0ccd027..bb20a62bf 100644 --- a/docs/ResponseFiles.md +++ b/docs/ResponseFiles.md @@ -27,10 +27,10 @@ Per the [MSVC docs](https://learn.microsoft.com/en-us/cpp/build/reference/cl-com 4. The `/link` directive has special treatment: 1. Entering an @file: if the `/link` option is provided prior to an `@file` in the command line, the `/link` directive does not affect any options within the `@file`. 2. Newlines: A `/link` directive provided in an `@file` on one line does not affect the next line. - 3. Exitting an @file: A `/link` directive on the final line of a response file does not affect options following the `@file` option in the command line. + 3. Exiting an @file: A `/link` directive on the final line of a response file does not affect options following the `@file` option in the command line. 5. A response file cannot contain additional `@file` options, they are not recursive. (found in a [separate doc](https://learn.microsoft.com/en-us/cpp/build/reference/at-specify-a-compiler-response-file?view=msvc-170)) 6. (implied) options can be wrapped in double-quotes (`"`), which allows whitespace to be preserved within the option -The msvc implementaion in sccache supports all of these **except** #4, because sccache doesn't accept the `/link` directive. +The msvc implementation in sccache supports all of these **except** #4, because sccache doesn't accept the `/link` directive. Additionally, because `msbuild` generates response files using an encoding other than `utf-8`, all text files under the [WHATWG encoding standard](https://encoding.spec.whatwg.org/) are supported. This includes both `utf-8` and `utf-16`. diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 796e2a47e..881738fef 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -18,7 +18,7 @@ description: | sandboxed compiler execution on build servers. See the distributed quickstart guide for more information. - sccache is also available as a GitHub Actions to faciliate the deployment + sccache is also available as a GitHub Actions to facilitate the deployment using GitHub Actions cache. website: https://github.com/mozilla/sccache contact: https://github.com/mozilla/sccache/issues diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index 248c9e195..c1dbcee22 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -1131,7 +1131,7 @@ impl<'a> Iterator for ExpandIncludeFile<'a> { loop { // Visit all arguments found in the most recently read response file. // Since response files are not recursive, we do not need to worry - // about these containing addditional @ directives. + // about these containing additional @ directives. if let Some(response_file_arg) = self.stack.pop() { return Some(response_file_arg); } @@ -1151,7 +1151,7 @@ impl<'a> Iterator for ExpandIncludeFile<'a> { Ok(content) => content, Err(err) => { debug!("failed to read @-file `{}`: {}", file_path.display(), err); - // If we failed to read the file content, return the orginal arg (including the `@` directive). + // If we failed to read the file content, return the original arg (including the `@` directive). return Some(arg); } }; diff --git a/src/compiler/nvhpc.rs b/src/compiler/nvhpc.rs index e86ce782e..a09eb6cac 100644 --- a/src/compiler/nvhpc.rs +++ b/src/compiler/nvhpc.rs @@ -132,7 +132,7 @@ impl CCompilerImpl for Nvhpc { trace!("preprocess"); let mut cmd = initialize_cmd_and_args(); - //NVHPC doesn't support disabling line info when outputing to console + //NVHPC doesn't support disabling line info when outputting to console cmd.arg("-E") .env_clear() .envs(env_vars.iter().map(|(k, v)| (k, v))) diff --git a/src/config.rs b/src/config.rs index 1e7bca567..07dee8b3f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -603,7 +603,7 @@ fn config_from_env() -> Result { if env::var("SCCACHE_GCS_OAUTH_URL").is_ok() { eprintln!("SCCACHE_GCS_OAUTH_URL has been deprecated"); - eprintln!("if you intend to use vm metadata for auth, please set correct service account intead"); + eprintln!("if you intend to use vm metadata for auth, please set correct service account instead"); } let credential_url = env::var("SCCACHE_GCS_CREDENTIALS_URL").ok(); diff --git a/tests/system.rs b/tests/system.rs index 8feab0fa6..c7716de25 100644 --- a/tests/system.rs +++ b/tests/system.rs @@ -583,7 +583,7 @@ fn test_cuda_compiles(compiler: &Compiler, tempdir: &Path) { assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); }); // By compiling another input source we verify that the pre-processor - // phase is correctly running and outputing text + // phase is correctly running and outputting text trace!("compile B"); sccache_command() .args(&compile_cuda_cmdline( From 45386750a48b6ed7daf9019db88a22dc7a7361c5 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sun, 19 Nov 2023 21:13:58 +0100 Subject: [PATCH 46/59] Revert "Activate preprocessor cache mode by default" Causes: https://github.com/mozilla/sccache/issues/1967 This reverts commit e63185b6d434aac581d5da1037f51786e60df2bd. --- docs/Local.md | 2 +- src/cache/cache.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Local.md b/docs/Local.md index 89525fbf0..5ccecf9f3 100644 --- a/docs/Local.md +++ b/docs/Local.md @@ -30,7 +30,7 @@ Preprocessor cache mode will be disabled if any of the following holds: Configuration options and their default values: -- `use_preprocessor_cache_mode`: `true`. Whether to use preprocessor cache mode entirely. +- `use_preprocessor_cache_mode`: `false`. Whether to use preprocessor cache mode entirely. - `file_stat_matches`: `false`. If false, only compare header files by hashing their contents. If true, will use size + ctime + mtime to check whether a file has changed. See other flags below for more control over this behavior. - `use_ctime_for_stat`: `true`. If true, uses the ctime (file status change on UNIX, creation time on Windows) to check that a file has/hasn't changed. Can be useful to disable when backdating modification times in a controlled manner. diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 37a690950..717cde09a 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -366,7 +366,7 @@ pub trait Storage: Send + Sync { /// Return the config for preprocessor cache mode if applicable fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { - // Enabled by default, only enabled in local mode + // Disabled by default, only enabled in local mode PreprocessorCacheModeConfig::default() } /// Return the preprocessor cache entry for a given preprocessor key, @@ -421,7 +421,7 @@ pub struct PreprocessorCacheModeConfig { impl Default for PreprocessorCacheModeConfig { fn default() -> Self { Self { - use_preprocessor_cache_mode: true, + use_preprocessor_cache_mode: false, file_stat_matches: false, use_ctime_for_stat: true, ignore_time_macros: false, From 7cca55a4ac24cab10cb63cf4e41542c96ccb7c9d Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 20 Nov 2023 09:20:52 +0100 Subject: [PATCH 47/59] prepare version 0.7.3 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b9c018b4..33b80503f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "sccache" -version = "0.7.2" +version = "0.7.3" dependencies = [ "anyhow", "ar", diff --git a/Cargo.toml b/Cargo.toml index 94e2b799d..e8b63e64b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" name = "sccache" rust-version = "1.67.1" -version = "0.7.2" +version = "0.7.3" categories = ["command-line-utilities", "development-tools::build-utils"] description = "Sccache is a ccache-like tool. It is used as a compiler wrapper and avoids compilation when possible, storing a cache in a remote storage using various cloud storage." From 68d0409108d9ff76003c0c644d9f5b252ba44120 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 22 Nov 2023 15:09:11 +0900 Subject: [PATCH 48/59] Fix some possible out-of-bound accesses in process_preprocesed_{file,line} --- src/compiler/c.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/compiler/c.rs b/src/compiler/c.rs index 0e0b1da0c..bf2261642 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -551,7 +551,7 @@ fn process_preprocessed_file( let mut normalized_include_paths: HashMap, Option>> = HashMap::new(); // There must be at least 7 characters (# 1 "x") left to potentially find an // include file path. - while start < total_len - 7 { + while start < total_len.saturating_sub(7) { let mut slice = &bytes[start..]; // Check if we look at a line containing the file name of an included file. // At least the following formats exist (where N is a positive integer): @@ -609,9 +609,12 @@ fn process_preprocessed_file( continue; } }; - } else if &bytes[start..start + INCBIN_DIRECTIVE.len()] == INCBIN_DIRECTIVE - && ((slice[7] == b' ' && (slice[8] == b'"' || (slice[8] == b'\\' && slice[9] == b'"'))) - || slice[7] == b'"') + } else if slice + .strip_prefix(INCBIN_DIRECTIVE) + .filter(|slice| { + slice.starts_with(b"\"") || slice.starts_with(b" \"") || slice.starts_with(b" \\\"") + }) + .is_some() { // An assembler .inc bin (without the space) statement, which could be // part of inline assembly, refers to an external file. If the file @@ -668,7 +671,7 @@ fn process_preprocessor_line( ) -> Result { let mut slice = &bytes[start..]; // Workarounds for preprocessor linemarker bugs in GCC version 6. - if slice[2] == b'3' { + if slice.get(2) == Some(&b'3') { if slice.starts_with(HASH_31_COMMAND_LINE_NEWLINE) { // Bogus extra line with #31, after the regular #1: // Ignore the whole line, and continue parsing. From e5fde1611848981561790aff636a625da8f28c79 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 23 Nov 2023 13:45:53 +0900 Subject: [PATCH 49/59] Fix test_parse_arguments_multiarch_cache_disabled The test fails when SCCACHE_CACHE_MULTIARCH is set, which can either happen externally, or when another tests sets it via with_var. with_var_unset prevents both (temp_env has a global lock specifically to avoid the problem with concurrent tests). Fixes #1982 --- src/compiler/gcc.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index e422d8735..e0c00c07b 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -931,7 +931,7 @@ mod test { use crate::mock_command::*; use crate::test::utils::*; - use temp_env::with_var; + use temp_env::{with_var, with_var_unset}; fn parse_arguments_( arguments: Vec, @@ -1768,18 +1768,20 @@ mod test { #[test] fn test_parse_arguments_multiarch_cache_disabled() { - assert_eq!( - CompilerArguments::CannotCache( - "multiple different -arch, and SCCACHE_CACHE_MULTIARCH not set", - None - ), - parse_arguments_( - stringvec![ - "-fPIC", "-arch", "arm64", "-arch", "i386", "-o", "foo.o", "-c", "foo.cpp" - ], - false + with_var_unset("SCCACHE_CACHE_MULTIARCH", || { + assert_eq!( + CompilerArguments::CannotCache( + "multiple different -arch, and SCCACHE_CACHE_MULTIARCH not set", + None + ), + parse_arguments_( + stringvec![ + "-fPIC", "-arch", "arm64", "-arch", "i386", "-o", "foo.o", "-c", "foo.cpp" + ], + false + ) ) - ) + }); } #[test] From d3d5cbe3035bc47177972059793cc3c4c8e69bf9 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 23 Nov 2023 09:00:36 +0900 Subject: [PATCH 50/59] Add support for `--` on clang/clang-cl command-line Fixes #1498 --- src/compiler/args.rs | 23 ++++- src/compiler/c.rs | 2 + src/compiler/diab.rs | 2 + src/compiler/gcc.rs | 158 +++++++++++++++++++++++++++++-- src/compiler/msvc.rs | 186 ++++++++++++++++++++++++++++++++----- src/compiler/tasking_vx.rs | 3 + 6 files changed, 341 insertions(+), 33 deletions(-) diff --git a/src/compiler/args.rs b/src/compiler/args.rs index e551c0f80..64d7eb98a 100644 --- a/src/compiler/args.rs +++ b/src/compiler/args.rs @@ -575,6 +575,7 @@ where { arguments: I, arg_info: S, + seen_double_dashes: Option, phantom: PhantomData, } @@ -592,9 +593,15 @@ where ArgsIter { arguments, arg_info, + seen_double_dashes: None, phantom: PhantomData, } } + + pub fn with_double_dashes(mut self) -> Self { + self.seen_double_dashes = Some(false); + self + } } impl Iterator for ArgsIter @@ -607,6 +614,14 @@ where fn next(&mut self) -> Option { if let Some(arg) = self.arguments.next() { + if let Some(seen_double_dashes) = &mut self.seen_double_dashes { + if !*seen_double_dashes && arg == "--" { + *seen_double_dashes = true; + } + if *seen_double_dashes { + return Some(Ok(Argument::Raw(arg))); + } + } let s = arg.to_string_lossy(); let arguments = &mut self.arguments; Some(match self.arg_info.search(&s[..]) { @@ -957,8 +972,11 @@ mod tests { "-plop", "-quxbar", // -quxbar is not -qux with a value of bar "-qux=value", + "--", + "non_flag", + "-flag-after-double-dashes", ]; - let iter = ArgsIter::new(args.iter().map(OsString::from), &ARGS[..]); + let iter = ArgsIter::new(args.iter().map(OsString::from), &ARGS[..]).with_double_dashes(); let expected = vec![ arg!(UnknownFlag("-nomatch")), arg!(WithValue("-foo", ArgData::Foo("value"), Separated)), @@ -979,6 +997,9 @@ mod tests { ArgData::Qux("value"), CanBeSeparated('=') )), + arg!(Raw("--")), + arg!(Raw("non_flag")), + arg!(Raw("-flag-after-double-dashes")), ]; match diff_with(iter, expected, |a, b| { assert_eq!(a.as_ref().unwrap(), b); diff --git a/src/compiler/c.rs b/src/compiler/c.rs index bf2261642..3aa9ea5a1 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -85,6 +85,8 @@ pub struct ArtifactDescriptor { pub struct ParsedArguments { /// The input source file. pub input: PathBuf, + /// Whether to prepend the input with `--` + pub double_dash_input: bool, /// The type of language used in the input source file. pub language: Language, /// The flag required to compile for the given language diff --git a/src/compiler/diab.rs b/src/compiler/diab.rs index 307ff17c0..3429d6a27 100644 --- a/src/compiler/diab.rs +++ b/src/compiler/diab.rs @@ -285,6 +285,7 @@ where CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input: false, language, compilation_flag, depfile: None, @@ -744,6 +745,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index e0c00c07b..99a71947d 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -254,6 +254,7 @@ where { let mut output_arg = None; let mut input_arg = None; + let mut double_dash_input = false; let mut dep_target = None; let mut dep_flag = OsString::from("-MT"); let mut common_args = vec![]; @@ -290,7 +291,11 @@ where let mut too_hard_for_preprocessor_cache_mode = false; - for arg in ArgsIter::new(it, arg_info) { + let mut args_iter = ArgsIter::new(it, arg_info); + if kind == CCompilerKind::Clang { + args_iter = args_iter.with_double_dashes(); + } + for arg in args_iter { let arg = try_or_cannot_cache!(arg, "argument parse"); // Check if the value part of this argument begins with '@'. If so, we either // failed to expand it, or it was a concatenated argument - either way, bail. @@ -392,6 +397,11 @@ where } Some(XClang(s)) => xclangs.push(s.clone()), None => match arg { + Argument::Raw(ref val) if val == "--" => { + if input_arg.is_none() { + double_dash_input = true; + } + } Argument::Raw(ref val) => { if input_arg.is_some() { multiple_input = true; @@ -605,6 +615,7 @@ where CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input, language, compilation_flag, depfile: None, @@ -703,11 +714,14 @@ fn preprocess_cmd( arch_args_to_use = &parsed_args.arch_args; } - cmd.arg(&parsed_args.input) - .args(&parsed_args.preprocessor_args) + cmd.args(&parsed_args.preprocessor_args) .args(&parsed_args.dependency_args) .args(&parsed_args.common_args) - .args(arch_args_to_use) + .args(arch_args_to_use); + if parsed_args.double_dash_input { + cmd.arg("--"); + } + cmd.arg(&parsed_args.input) .env_clear() .envs(env_vars.iter().map(|(k, v)| (k, v))) .current_dir(cwd); @@ -780,7 +794,6 @@ pub fn generate_compile_commands( } arguments.extend(vec![ parsed_args.compilation_flag.clone(), - parsed_args.input.clone().into(), "-o".into(), out_file.into(), ]); @@ -788,6 +801,10 @@ pub fn generate_compile_commands( arguments.extend(parsed_args.unhashed_args.clone()); arguments.extend(parsed_args.common_args.clone()); arguments.extend(parsed_args.arch_args.clone()); + if parsed_args.double_dash_input { + arguments.push("--".into()); + } + arguments.push(parsed_args.input.clone().into()); let command = CompileCommand { executable: executable.to_owned(), arguments, @@ -941,6 +958,20 @@ mod test { parse_arguments(&args, ".".as_ref(), &ARGS[..], plusplus, CCompilerKind::Gcc) } + fn parse_arguments_clang( + arguments: Vec, + plusplus: bool, + ) -> CompilerArguments { + let args = arguments.iter().map(OsString::from).collect::>(); + parse_arguments( + &args, + ".".as_ref(), + &ARGS[..], + plusplus, + CCompilerKind::Clang, + ) + } + #[test] fn test_parse_arguments_simple() { let args = stringvec!["-c", "foo.c", "-o", "foo.o"]; @@ -1336,6 +1367,65 @@ mod test { assert!(!msvc_show_includes); } + #[test] + fn test_parse_arguments_double_dash() { + let args = stringvec!["-c", "-o", "foo.o", "--", "foo.c"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_(args.clone(), false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // GCC doesn't support double dashes. If we got one, we'll pass them + // through to GCC for it to error out. + assert!(!double_dash_input); + assert_eq!(ovec!["--"], common_args); + + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + assert!(double_dash_input); + assert!(common_args.is_empty()); + + let args = stringvec!["-c", "-o", "foo.o", "foo.c", "--"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // Double dash after input file is ignored. + assert!(!double_dash_input); + assert!(common_args.is_empty()); + + let args = stringvec!["-c", "-o", "foo.o", "foo.c", "--", "bar.c"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"bar.c\"]".to_string())), + parse_arguments_clang(args, false) + ); + + let args = stringvec!["-c", "-o", "foo.o", "foo.c", "--", "-fPIC"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"-fPIC\"]".to_string())), + parse_arguments_clang(args, false) + ); + } + #[test] fn test_parse_arguments_explicit_dep_target() { let args = @@ -1503,9 +1593,9 @@ mod test { "c++", "-E", "-fdirectives-only", - "foo.cc", "-D__arm64__=1", - "-D__i386__=1" + "-D__i386__=1", + "foo.cc" ]; assert_eq!(cmd.args, expected_args); }); @@ -1538,13 +1628,38 @@ mod test { "c++", "-E", "-fdirectives-only", - "foo.cc", "-arch", - "arm64" + "arm64", + "foo.cc" ]; assert_eq!(cmd.args, expected_args); } + #[test] + fn test_preprocess_double_dash_input() { + let args = stringvec!["-c", "-o", "foo.o", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let mut cmd = MockCommand { + child: None, + args: vec![], + }; + preprocess_cmd( + &mut cmd, + &parsed_args, + Path::new(""), + &[], + true, + CCompilerKind::Clang, + true, + vec![], + ); + let expected_args = ovec!["-x", "c", "-E", "-frewrite-includes", "--", "foo.c"]; + assert_eq!(cmd.args, expected_args); + } + #[test] fn pedantic_default() { let args = stringvec!["-pedantic", "-c", "foo.cc"]; @@ -1890,6 +2005,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, @@ -1938,6 +2054,30 @@ mod test { assert_eq!(0, creator.lock().unwrap().children.len()); } + #[test] + fn test_compile_double_dash_input() { + let args = stringvec!["-c", "-o", "foo.o", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args, false) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let f = TestFixture::new(); + let compiler = &f.bins[0]; + let mut path_transformer = dist::PathTransformer::default(); + let (command, _, _) = generate_compile_commands( + &mut path_transformer, + compiler, + &parsed_args, + f.tempdir.path(), + &[], + CCompilerKind::Clang, + false, + ) + .unwrap(); + let expected_args = ovec!["-x", "c", "-c", "-o", "foo.o", "--", "foo.c"]; + assert_eq!(command.arguments, expected_args); + } + #[test] fn test_parse_arguments_plusplus() { let args = stringvec!["-c", "foo.c", "-o", "foo.o"]; diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index c1dbcee22..81336d744 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -492,6 +492,7 @@ pub fn parse_arguments( ) -> CompilerArguments { let mut output_arg = None; let mut input_arg = None; + let mut double_dash_input = false; let mut common_args = vec![]; let mut unhashed_args = vec![]; let mut preprocessor_args = vec![]; @@ -512,7 +513,10 @@ pub fn parse_arguments( // Custom iterator to expand `@` arguments which stand for reading a file // and interpreting it as a list of more arguments. let it = ExpandIncludeFile::new(cwd, arguments); - let it = ArgsIter::new(it, (&ARGS[..], &SLASH_ARGS[..])); + let mut it = ArgsIter::new(it, (&ARGS[..], &SLASH_ARGS[..])); + if is_clang { + it = it.with_double_dashes(); + } for arg in it { let arg = try_or_cannot_cache!(arg, "argument parse"); match arg.get_data() { @@ -553,6 +557,11 @@ pub fn parse_arguments( Some(Clang(s)) => clangs.push(s.clone()), None => { match arg { + Argument::Raw(ref val) if val == "--" => { + if input_arg.is_none() { + double_dash_input = true; + } + } Argument::Raw(ref val) => { if input_arg.is_some() { // Can't cache compilations with multiple inputs. @@ -797,6 +806,7 @@ pub fn parse_arguments( CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input, language, compilation_flag, depfile, @@ -852,22 +862,17 @@ fn normpath(path: &str) -> String { } #[allow(clippy::too_many_arguments)] -pub async fn preprocess( - creator: &T, - executable: &Path, +pub fn preprocess_cmd( + cmd: &mut T, parsed_args: &ParsedArguments, cwd: &Path, env_vars: &[(OsString, OsString)], may_dist: bool, - includes_prefix: &str, rewrite_includes_only: bool, is_clang: bool, -) -> Result -where - T: CommandCreatorSync, +) where + T: RunCommand, { - let mut cmd = creator.clone().new_command_sync(executable); - // When performing distributed compilation, line number info is important for error // reporting and to not cause spurious compilation failure (e.g. no exceptions build // fails due to exceptions transitively included in the stdlib). @@ -880,8 +885,7 @@ where cmd.arg("-EP"); } - cmd.arg(&parsed_args.input) - .arg("-nologo") + cmd.arg("-nologo") .args(&parsed_args.preprocessor_args) .args(&parsed_args.dependency_args) .args(&parsed_args.common_args) @@ -908,6 +912,38 @@ where cmd.arg("-clang:-frewrite-includes"); } + if parsed_args.double_dash_input { + cmd.arg("--"); + } + cmd.arg(&parsed_args.input); +} + +#[allow(clippy::too_many_arguments)] +pub async fn preprocess( + creator: &T, + executable: &Path, + parsed_args: &ParsedArguments, + cwd: &Path, + env_vars: &[(OsString, OsString)], + may_dist: bool, + includes_prefix: &str, + rewrite_includes_only: bool, + is_clang: bool, +) -> Result +where + T: CommandCreatorSync, +{ + let mut cmd = creator.clone().new_command_sync(executable); + preprocess_cmd( + &mut cmd, + parsed_args, + cwd, + env_vars, + may_dist, + rewrite_includes_only, + is_clang, + ); + if log_enabled!(Debug) { debug!("preprocess: {:?}", cmd); } @@ -1013,16 +1049,15 @@ fn generate_compile_commands( let mut fo = OsString::from("-Fo"); fo.push(out_file); - let mut arguments: Vec = vec![ - parsed_args.compilation_flag.clone(), - parsed_args.input.clone().into(), - fo, - ]; + let mut arguments: Vec = vec![parsed_args.compilation_flag.clone(), fo]; arguments.extend(parsed_args.preprocessor_args.clone()); arguments.extend(parsed_args.dependency_args.clone()); arguments.extend(parsed_args.unhashed_args.clone()); arguments.extend(parsed_args.common_args.clone()); - + if parsed_args.double_dash_input { + arguments.push("--".into()); + } + arguments.push(parsed_args.input.clone().into()); let command = CompileCommand { executable: executable.to_owned(), arguments, @@ -1039,17 +1074,19 @@ fn generate_compile_commands( let mut fo = String::from("-Fo"); fo.push_str(&path_transformer.as_dist(out_file)?); - let mut arguments: Vec = vec![ - parsed_args.compilation_flag.clone().into_string().ok()?, - path_transformer.as_dist(&parsed_args.input)?, - fo, - ]; + let mut arguments: Vec = + vec![parsed_args.compilation_flag.clone().into_string().ok()?, fo]; // It's important to avoid preprocessor_args because of things like /FI which // forcibly includes another file. This does mean we're potentially vulnerable // to misidentification of flags like -DYNAMICBASE (though in that specific // case we're safe as it only applies to link time, which sccache avoids). arguments.extend(dist::osstrings_to_strings(&parsed_args.common_args)?); + if parsed_args.double_dash_input { + arguments.push("--".into()); + } + arguments.push(path_transformer.as_dist(&parsed_args.input)?); + Some(dist::CompileCommand { executable: path_transformer.as_dist(executable)?, arguments, @@ -1321,6 +1358,10 @@ mod test { super::parse_arguments(&arguments, &std::env::current_dir().unwrap(), false) } + fn parse_arguments_clang(arguments: Vec) -> CompilerArguments { + super::parse_arguments(&arguments, &std::env::current_dir().unwrap(), true) + } + #[test] fn test_detect_showincludes_prefix() { drop(env_logger::try_init()); @@ -1510,6 +1551,65 @@ mod test { assert!(!msvc_show_includes); } + #[test] + fn test_parse_arguments_double_dash() { + let args = ovec!["-c", "-Fofoo.obj", "--", "foo.c"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments(args.clone()) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // MSVC doesn't support double dashes. If we got one, we'll pass them + // through to MSVC for it to error out. + assert!(!double_dash_input); + assert_eq!(ovec!["--"], common_args); + + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + assert!(double_dash_input); + assert!(common_args.is_empty()); + + let args = ovec!["-c", "-Fofoo.obj", "foo.c", "--"]; + let ParsedArguments { + input, + double_dash_input, + common_args, + .. + } = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + assert_eq!(Some("foo.c"), input.to_str()); + // Double dash after input file is ignored. + assert!(!double_dash_input); + assert!(common_args.is_empty()); + + let args = ovec!["-c", "-Fofoo.obj", "foo.c", "--", "bar.c"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"bar.c\"]".to_string())), + parse_arguments_clang(args) + ); + + let args = ovec!["-c", "-Fofoo.obj", "foo.c", "--", "-fPIC"]; + assert_eq!( + CompilerArguments::CannotCache("multiple input files", Some("[\"-fPIC\"]".to_string())), + parse_arguments_clang(args) + ); + } + #[test] fn parse_argument_slashes() { let args = ovec!["-c", "foo.c", "/Fofoo.obj"]; @@ -2241,12 +2341,29 @@ mod test { ); } + #[test] + fn test_preprocess_double_dash_input() { + let args = ovec!["-c", "-Fofoo.o.bj", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let mut cmd = MockCommand { + child: None, + args: vec![], + }; + preprocess_cmd(&mut cmd, &parsed_args, Path::new(""), &[], true, true, true); + let expected_args = ovec!["-E", "-nologo", "-clang:-frewrite-includes", "--", "foo.c"]; + assert_eq!(cmd.args, expected_args); + } + #[test] fn test_compile_simple() { let creator = new_creator(); let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, @@ -2293,6 +2410,28 @@ mod test { assert_eq!(0, creator.lock().unwrap().children.len()); } + #[test] + fn test_compile_double_dash_input() { + let args = ovec!["-c", "-Fofoo.obj", "--", "foo.c"]; + let parsed_args = match parse_arguments_clang(args) { + CompilerArguments::Ok(args) => args, + o => panic!("Got unexpected parse result: {:?}", o), + }; + let f = TestFixture::new(); + let compiler = &f.bins[0]; + let mut path_transformer = dist::PathTransformer::default(); + let (command, _, _) = generate_compile_commands( + &mut path_transformer, + compiler, + &parsed_args, + f.tempdir.path(), + &[], + ) + .unwrap(); + let expected_args = ovec!["-c", "-Fofoo.obj", "--", "foo.c"]; + assert_eq!(command.arguments, expected_args); + } + #[test] fn test_compile_not_cacheable_pdb() { let creator = new_creator(); @@ -2300,6 +2439,7 @@ mod test { let pdb = f.touch("foo.pdb").unwrap(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "/c".into(), depfile: None, diff --git a/src/compiler/tasking_vx.rs b/src/compiler/tasking_vx.rs index 1bd4a02a1..ee3d514a6 100644 --- a/src/compiler/tasking_vx.rs +++ b/src/compiler/tasking_vx.rs @@ -266,6 +266,7 @@ where CompilerArguments::Ok(ParsedArguments { input: input.into(), + double_dash_input: false, language, compilation_flag: "-c".into(), depfile, @@ -683,6 +684,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.c".into(), + double_dash_input: false, language: Language::C, compilation_flag: "-c".into(), depfile: None, @@ -731,6 +733,7 @@ mod test { let f = TestFixture::new(); let parsed_args = ParsedArguments { input: "foo.cu".into(), + double_dash_input: false, language: Language::Cuda, compilation_flag: "-c".into(), depfile: None, From 7ce78106107deb250aa7c6519e70adeb3284ffe9 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 23 Nov 2023 16:31:03 +0900 Subject: [PATCH 51/59] Remove uses of Lazy> Mutex::new has been const since 1.63.0. --- src/config.rs | 2 +- src/dist/client_auth.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/config.rs b/src/config.rs index 07dee8b3f..ed1d38ba2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -37,7 +37,7 @@ pub use crate::cache::PreprocessorCacheModeConfig; use crate::errors::*; static CACHED_CONFIG_PATH: Lazy = Lazy::new(CachedConfig::file_config_path); -static CACHED_CONFIG: Lazy>> = Lazy::new(|| Mutex::new(None)); +static CACHED_CONFIG: Mutex> = Mutex::new(None); const ORGANIZATION: &str = "Mozilla"; const APP_NAME: &str = "sccache"; diff --git a/src/dist/client_auth.rs b/src/dist/client_auth.rs index 573de1dea..cfe506106 100644 --- a/src/dist/client_auth.rs +++ b/src/dist/client_auth.rs @@ -89,7 +89,6 @@ mod code_grant_pkce { use base64::Engine; use futures::channel::oneshot; use hyper::{Body, Method, Request, Response, StatusCode}; - use once_cell::sync::Lazy; use rand::{rngs::OsRng, RngCore}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -142,7 +141,7 @@ mod code_grant_pkce { pub shutdown_tx: Option>, } - pub static STATE: Lazy>> = Lazy::new(|| Mutex::new(None)); + pub static STATE: Mutex> = Mutex::new(None); pub fn generate_verifier_and_challenge() -> Result<(String, String)> { let mut code_verifier_bytes = vec![0; NUM_CODE_VERIFIER_BYTES]; @@ -279,7 +278,6 @@ mod implicit { }; use futures::channel::oneshot; use hyper::{Body, Method, Request, Response, StatusCode}; - use once_cell::sync::Lazy; use std::collections::HashMap; use std::sync::mpsc; use std::sync::Mutex; @@ -308,7 +306,7 @@ mod implicit { pub shutdown_tx: Option>, } - pub static STATE: Lazy>> = Lazy::new(|| Mutex::new(None)); + pub static STATE: Mutex> = Mutex::new(None); pub fn finish_url(client_id: &str, url: &mut Url, redirect_uri: &str, state: &str) { url.query_pairs_mut() From 240c38f4df9df7bd4d0ce52fc73241fe277fc481 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 23 Nov 2023 05:17:26 +0900 Subject: [PATCH 52/59] Catch panics in the server and transmit them as errors to the client. Currently, when the server panics, the client never gets a response, and the compilation gets stuck. The only indication of something going wrong is in the logs if they are enabled. With this change, the error is transmitted to the client, which then fails and prints it. For instance, one of the out-of-bound accesses fixed in #1980 now prints this on the client side before returning a non-zero exit code: sccache: encountered fatal error sccache: error: thread 'tokio-runtime-worker' panicked at src/compiler/c.rs:612:25: range end index 7 out of range for slice of length 1 sccache: caused by: thread 'tokio-runtime-worker' panicked at src/compiler/c.rs:612:25: range end index 7 out of range for slice of length 1 Fixes #756 --- src/server.rs | 59 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/src/server.rs b/src/server.rs index 239c96994..562fc39ba 100644 --- a/src/server.rs +++ b/src/server.rs @@ -36,6 +36,7 @@ use futures::future::FutureExt; use futures::{future, stream, Sink, SinkExt, Stream, StreamExt, TryFutureExt}; use number_prefix::NumberPrefix; use serde::{Deserialize, Serialize}; +use std::cell::Cell; use std::collections::HashMap; use std::env; use std::ffi::{OsStr, OsString}; @@ -392,12 +393,29 @@ impl DistClientContainer { } } +thread_local! { + /// catch_unwind doesn't provide panic location, so we store that + /// information via a panic hook to be used when catch_unwind + /// catches a panic. + static PANIC_LOCATION: Cell> = Cell::new(None); +} + /// Start an sccache server, listening on `port`. /// /// Spins an event loop handling client connections until a client /// requests a shutdown. pub fn start_server(config: &Config, port: u16) -> Result<()> { info!("start_server: port: {}", port); + let panic_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + PANIC_LOCATION.with(|l| { + l.set( + info.location() + .map(|loc| (loc.file().to_string(), loc.line(), loc.column())), + ) + }); + panic_hook(info) + })); let client = unsafe { Client::new() }; let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -1182,20 +1200,35 @@ where let task = async move { let dist_client = me.dist_client.get_client().await; let result = match dist_client { - Ok(client) => { - hasher - .get_cached_or_compile( - client, - creator, - storage, - arguments, - cwd, - env_vars, - cache_control, - pool, + Ok(client) => std::panic::AssertUnwindSafe(hasher.get_cached_or_compile( + client, + creator, + storage, + arguments, + cwd, + env_vars, + cache_control, + pool, + )) + .catch_unwind() + .await + .map_err(|e| { + let panic = e + .downcast_ref::<&str>() + .map(|s| &**s) + .or_else(|| e.downcast_ref::().map(|s| &**s)) + .unwrap_or("An unknown panic was caught."); + let thread = std::thread::current(); + let thread_name = thread.name().unwrap_or("unnamed"); + if let Some((file, line, column)) = PANIC_LOCATION.with(|l| l.take()) { + anyhow!( + "thread '{thread_name}' panicked at {file}:{line}:{column}: {panic}" ) - .await - } + } else { + anyhow!("thread '{thread_name}' panicked: {panic}") + } + }) + .and_then(std::convert::identity), Err(e) => Err(e), }; let mut cache_write = None; From aa20391f229ccba4c038cfe6412bae184fb6fc3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Gom=C3=A8s?= Date: Mon, 20 Nov 2023 15:42:49 +0100 Subject: [PATCH 53/59] Only activate direct mode by default for local 45386750a48b6ed7daf9019db88a22dc7a7361c5 reverted my initial change. This re-enables direct mode by default, making sure only the disk cache is affected. There may still be an issue with Windows, but details are sparse and no reproduction has been found yet. --- docs/Local.md | 2 +- src/cache/cache.rs | 12 +++++++++++- src/config.rs | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/Local.md b/docs/Local.md index 5ccecf9f3..89525fbf0 100644 --- a/docs/Local.md +++ b/docs/Local.md @@ -30,7 +30,7 @@ Preprocessor cache mode will be disabled if any of the following holds: Configuration options and their default values: -- `use_preprocessor_cache_mode`: `false`. Whether to use preprocessor cache mode entirely. +- `use_preprocessor_cache_mode`: `true`. Whether to use preprocessor cache mode entirely. - `file_stat_matches`: `false`. If false, only compare header files by hashing their contents. If true, will use size + ctime + mtime to check whether a file has changed. See other flags below for more control over this behavior. - `use_ctime_for_stat`: `true`. If true, uses the ctime (file status change on UNIX, creation time on Windows) to check that a file has/hasn't changed. Can be useful to disable when backdating modification times in a controlled manner. diff --git a/src/cache/cache.rs b/src/cache/cache.rs index 717cde09a..c31761223 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -366,7 +366,7 @@ pub trait Storage: Send + Sync { /// Return the config for preprocessor cache mode if applicable fn preprocessor_cache_mode_config(&self) -> PreprocessorCacheModeConfig { - // Disabled by default, only enabled in local mode + // Enable by default, only in local mode PreprocessorCacheModeConfig::default() } /// Return the preprocessor cache entry for a given preprocessor key, @@ -431,6 +431,16 @@ impl Default for PreprocessorCacheModeConfig { } } +impl PreprocessorCacheModeConfig { + /// Return a default [`Self`], but with the cache active. + pub fn activated() -> Self { + Self { + use_preprocessor_cache_mode: true, + ..Default::default() + } + } +} + /// Implement storage for operator. #[cfg(any(feature = "s3", feature = "azure", feature = "gcs", feature = "redis"))] #[async_trait] diff --git a/src/config.rs b/src/config.rs index ed1d38ba2..53d90c37f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -169,7 +169,7 @@ impl Default for DiskCacheConfig { DiskCacheConfig { dir: default_disk_cache_dir(), size: default_disk_cache_size(), - preprocessor_cache_mode: Default::default(), + preprocessor_cache_mode: PreprocessorCacheModeConfig::activated(), } } } @@ -1296,7 +1296,7 @@ token = "webdavtoken" disk: Some(DiskCacheConfig { dir: PathBuf::from("/tmp/.cache/sccache"), size: 7 * 1024 * 1024 * 1024, - preprocessor_cache_mode: Default::default(), + preprocessor_cache_mode: PreprocessorCacheModeConfig::activated(), }), gcs: Some(GCSCacheConfig { bucket: "bucket".to_owned(), From 2dfbe0409dc3b2b9a67ff2b4f9d88f281ed1bfc2 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 23 Nov 2023 16:15:46 +0900 Subject: [PATCH 54/59] Remove some unnecessary clones --- src/compiler/diab.rs | 6 +++--- src/compiler/gcc.rs | 8 ++++---- src/compiler/msvc.rs | 8 ++++---- src/compiler/tasking_vx.rs | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/compiler/diab.rs b/src/compiler/diab.rs index 3429d6a27..7ef453b29 100644 --- a/src/compiler/diab.rs +++ b/src/compiler/diab.rs @@ -352,9 +352,9 @@ pub fn generate_compile_commands( "-o".into(), out_file.into(), ]; - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); let command = CompileCommand { executable: executable.to_owned(), arguments, diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index 99a71947d..0bd2e224b 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -797,10 +797,10 @@ pub fn generate_compile_commands( "-o".into(), out_file.into(), ]); - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); - arguments.extend(parsed_args.arch_args.clone()); + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); + arguments.extend_from_slice(&parsed_args.arch_args); if parsed_args.double_dash_input { arguments.push("--".into()); } diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index 81336d744..56b2260d2 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -1050,10 +1050,10 @@ fn generate_compile_commands( fo.push(out_file); let mut arguments: Vec = vec![parsed_args.compilation_flag.clone(), fo]; - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.dependency_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.dependency_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); if parsed_args.double_dash_input { arguments.push("--".into()); } diff --git a/src/compiler/tasking_vx.rs b/src/compiler/tasking_vx.rs index ee3d514a6..09cc8d74a 100644 --- a/src/compiler/tasking_vx.rs +++ b/src/compiler/tasking_vx.rs @@ -367,9 +367,9 @@ fn generate_compile_commands( "-o".into(), out_file.path.as_os_str().into(), ]; - arguments.extend(parsed_args.preprocessor_args.clone()); - arguments.extend(parsed_args.unhashed_args.clone()); - arguments.extend(parsed_args.common_args.clone()); + arguments.extend_from_slice(&parsed_args.preprocessor_args); + arguments.extend_from_slice(&parsed_args.unhashed_args); + arguments.extend_from_slice(&parsed_args.common_args); let command = CompileCommand { executable: executable.to_owned(), arguments, From 6ff88eb4505937222bfbf09a064e3810dfe74440 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Fri, 24 Nov 2023 09:47:14 +0900 Subject: [PATCH 55/59] Activate direct mode when e.g. SCCACHE_DIR is set. --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 53d90c37f..a88a61e0d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -710,7 +710,7 @@ fn config_from_env() -> Result { .ok() .and_then(|v| parse_size(&v)); - let mut preprocessor_mode_config = PreprocessorCacheModeConfig::default(); + let mut preprocessor_mode_config = PreprocessorCacheModeConfig::activated(); let preprocessor_mode_overridden = match env::var("SCCACHE_DIRECT").as_deref() { Ok("on") | Ok("true") => { preprocessor_mode_config.use_preprocessor_cache_mode = true; From fb0ab0c42268c2ea9bbc1ec9bc848330cfdf08ec Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Mon, 27 Nov 2023 10:15:33 +0100 Subject: [PATCH 56/59] Prepare version 0.7.4 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33b80503f..29d5e41fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "sccache" -version = "0.7.3" +version = "0.7.4" dependencies = [ "anyhow", "ar", diff --git a/Cargo.toml b/Cargo.toml index e8b63e64b..d7789918c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ edition = "2021" name = "sccache" rust-version = "1.67.1" -version = "0.7.3" +version = "0.7.4" categories = ["command-line-utilities", "development-tools::build-utils"] description = "Sccache is a ccache-like tool. It is used as a compiler wrapper and avoids compilation when possible, storing a cache in a remote storage using various cloud storage." From 9abbbac3cdc8c9a5492e46ae8be7c21d8a69d4ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:25:19 +0000 Subject: [PATCH 57/59] build(deps): bump serde from 1.0.190 to 1.0.193 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.190 to 1.0.193. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.190...v1.0.193) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29d5e41fb..697b9d06f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2465,18 +2465,18 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.190" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", From a1acbda284582bd8083403d1b29141f21cb1f774 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Tue, 5 Dec 2023 02:59:06 +0100 Subject: [PATCH 58/59] Bump freebsd-vm action to v1.0.2 & use ubuntu (#1994) Copied from https://github.com/uutils/coreutils/pull/5606/files --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d00c9a65..c40de4ccc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,15 +264,15 @@ jobs: fail-fast: false matrix: job: - - { os: macos-12 } - release: ["13.2"] + - { os: ubuntu-22.04 } steps: - uses: actions/checkout@v4 - name: Prepare, build and test - uses: vmactions/freebsd-vm@v0 + uses: vmactions/freebsd-vm@v1.0.2 with: mem: 8192 usesh: true + sync: rsync copyback: false prepare: pkg install -y ca_root_nss curl gmake gtar pot sudo run: | @@ -289,7 +289,7 @@ jobs: TEST_USER=tester TEST_USER_HOME="/opt/$TEST_USER" REPO_NAME=${GITHUB_WORKSPACE##*/} - WORKSPACE_PARENT="/Users/runner/work/${REPO_NAME}" + WORKSPACE_PARENT="/home/runner/work/${REPO_NAME}" WORKSPACE="${WORKSPACE_PARENT}/${REPO_NAME}" export WORKSPACE # From 920564c129acfb5e333b71d795261256332bb191 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 17:31:47 +0000 Subject: [PATCH 59/59] build(deps): bump nix from 0.26.2 to 0.26.4 Bumps [nix](https://github.com/nix-rust/nix) from 0.26.2 to 0.26.4. - [Changelog](https://github.com/nix-rust/nix/blob/v0.26.4/CHANGELOG.md) - [Commits](https://github.com/nix-rust/nix/compare/v0.26.2...v0.26.4) --- updated-dependencies: - dependency-name: nix dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 13 +++---------- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 697b9d06f..fd2d6e76a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1505,16 +1505,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", "libc", "memoffset", "pin-utils", - "static_assertions", ] [[package]] @@ -2366,7 +2365,7 @@ dependencies = [ "memchr", "memmap2", "mime", - "nix 0.26.2", + "nix 0.26.4", "num_cpus", "number_prefix", "object", @@ -2661,12 +2660,6 @@ dependencies = [ "der", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "stringmatch" version = "0.3.3" diff --git a/Cargo.toml b/Cargo.toml index d7789918c..015861258 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,7 +107,7 @@ zstd = "0.13" # dist-server only memmap2 = "0.9.0" -nix = { version = "0.26.2", optional = true } +nix = { version = "0.26.4", optional = true } object = "0.30" rouille = { version = "3.6", optional = true, default-features = false, features = [ "ssl",