Skip to content

Commit 5607dd9

Browse files
ElectreAASElectreAAS
authored
Enable dune cache by default (#10710)
* Added feature flag to enable dune cache by default
1 parent 5d05171 commit 5607dd9

27 files changed

+123
-30
lines changed

‎bin/common.ml

+35-1
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,7 @@ module Builder = struct
583583
; file_watcher : Dune_engine.Scheduler.Run.file_watcher
584584
; workspace_config : Dune_rules.Workspace.Clflags.t
585585
; cache_debug_flags : Dune_engine.Cache_debug_flags.t
586+
; cache_rules_default : bool
586587
; report_errors_config : Dune_engine.Report_errors_config.t
587588
; separate_error_messages : bool
588589
; stop_on_first_error : bool
@@ -932,6 +933,20 @@ module Builder = struct
932933
useful for Dune developers to make Dune tests of the digest cache more \
933934
reproducible.")
934935
and+ cache_debug_flags = cache_debug_flags_term
936+
and+ cache_rules_default =
937+
let default =
938+
Dune_lang.Toggle.of_bool !Dune_engine.Clflags.can_go_in_shared_cache_default
939+
in
940+
let doc =
941+
Printf.sprintf
942+
"Enable or disable caching rules (%s). Default is `%s'."
943+
(Arg.doc_alts_enum Config.Toggle.all)
944+
(Config.Toggle.to_string default)
945+
in
946+
Arg.(
947+
value
948+
& opt (enum Config.Toggle.all) default
949+
& info [ "cache-rules" ] ~docs ~env:(Cmd.Env.info ~doc "DUNE_CACHE_RULES") ~doc)
935950
and+ report_errors_config =
936951
Arg.(
937952
value
@@ -1008,6 +1023,7 @@ module Builder = struct
10081023
; config_from_config_file
10091024
}
10101025
; cache_debug_flags
1026+
; cache_rules_default = Dune_lang.Toggle.enabled cache_rules_default
10111027
; report_errors_config
10121028
; separate_error_messages
10131029
; stop_on_first_error
@@ -1163,6 +1179,23 @@ let build (builder : Builder.t) =
11631179
{ builder; root; rpc; stats }
11641180
;;
11651181

1182+
let maybe_init_cache (cache_config : Dune_cache.Config.t) =
1183+
match cache_config with
1184+
| Disabled -> cache_config
1185+
| Enabled _ ->
1186+
(match Dune_cache_storage.Layout.create_cache_directories () with
1187+
| Ok () -> cache_config
1188+
| Error (path, exn) ->
1189+
User_warning.emit
1190+
~hints:
1191+
[ Pp.textf "Make sure the directory %s can be created" (Path.to_string path) ]
1192+
[ Pp.textf
1193+
"Cache directories could not be created: %s; disabling cache"
1194+
(Unix.error_message exn)
1195+
];
1196+
Disabled)
1197+
;;
1198+
11661199
let init (builder : Builder.t) =
11671200
let c = build builder in
11681201
if c.root.dir <> Filename.current_dir_name then Sys.chdir c.root.dir;
@@ -1216,7 +1249,7 @@ let init (builder : Builder.t) =
12161249
Dune_rules.Main.init
12171250
~stats:c.stats
12181251
~sandboxing_preference:config.sandboxing_preference
1219-
~cache_config
1252+
~cache_config:(maybe_init_cache cache_config)
12201253
~cache_debug_flags:c.builder.cache_debug_flags
12211254
();
12221255
Only_packages.Clflags.set c.builder.only_packages;
@@ -1241,6 +1274,7 @@ let init (builder : Builder.t) =
12411274
Dune_rules.Clflags.ignore_lock_dir := c.builder.ignore_lock_dir;
12421275
Dune_rules.Clflags.on_missing_dune_project_file
12431276
:= if c.builder.require_dune_project_file then Error else Warn;
1277+
Dune_engine.Clflags.can_go_in_shared_cache_default := c.builder.cache_rules_default;
12441278
Log.info
12451279
[ Pp.textf
12461280
"Workspace root: %s"

‎src/dune_cache_storage/layout.ml

+5-3
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,9 @@ let value_storage_dir = Versioned.value_storage_dir Version.Value.current
8282
let value_path = Versioned.value_path Version.Value.current
8383

8484
let create_cache_directories () =
85-
List.iter
86-
[ temp_dir; metadata_storage_dir; file_storage_dir; value_storage_dir ]
87-
~f:(fun path -> ignore (Fpath.mkdir_p (Path.to_string path) : Fpath.mkdir_p_result))
85+
[ temp_dir; metadata_storage_dir; file_storage_dir; value_storage_dir ]
86+
|> Result.List.iter ~f:(fun path ->
87+
match Fpath.mkdir_p (Path.to_string path) with
88+
| Already_exists | Created -> Ok ()
89+
| exception Unix.Unix_error (e, _, _) -> Error (path, e))
8890
;;

‎src/dune_cache_storage/layout.mli

+5-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,11 @@ open Import
1212
val root_dir : Path.t
1313

1414
(** Create a few subdirectories in [root_dir]. We expose this function because
15-
we don't want to modify the file system when the cache is disabled. *)
16-
val create_cache_directories : unit -> unit
15+
we don't want to modify the file system when the cache is disabled.
16+
17+
Returns whether creation has succeeded or in the case of error which directory
18+
could not be created. *)
19+
val create_cache_directories : unit -> (unit, Path.t * Unix.error) result
1720

1821
(** This directory stores metadata files, one per each historically executed
1922
build rule or output-producing action. (While this is a convenient mental

‎src/dune_config_file/dune_config_file.ml

+2-2
Original file line numberDiff line numberDiff line change
@@ -289,9 +289,9 @@ module Dune_config = struct
289289
; concurrency = (if Execution_env.inside_dune then Fixed 1 else Auto)
290290
; terminal_persistence = Clear_on_rebuild
291291
; sandboxing_preference = []
292-
; cache_enabled = `Disabled
292+
; cache_enabled = `Enabled
293293
; cache_reproducibility_check = Skip
294-
; cache_storage_mode = None
294+
; cache_storage_mode = Some (Dune_cache_storage.Mode.default ())
295295
; action_stdout_on_success = Print
296296
; action_stderr_on_success = Print
297297
; experimental = []

‎src/dune_engine/action.ml

+1-1
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ module Full = struct
336336
let make
337337
?(env = Env.empty)
338338
?(locks = [])
339-
?(can_go_in_shared_cache = true)
339+
?(can_go_in_shared_cache = !Clflags.can_go_in_shared_cache_default)
340340
?(sandbox = Sandbox_config.default)
341341
action
342342
=

‎src/dune_engine/action.mli

+2-1
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,8 @@ module Full : sig
152152
val make
153153
: ?env:Env.t (** default [Env.empty] *)
154154
-> ?locks:Path.t list (** default [[]] *)
155-
-> ?can_go_in_shared_cache:bool (** default [true] *)
155+
-> ?can_go_in_shared_cache:bool
156+
(** default [!Clflags.can_fo_in_shared_cache_default] *)
156157
-> ?sandbox:Sandbox_config.t (** default [Sandbox_config.default] *)
157158
-> action
158159
-> t

‎src/dune_engine/build_config.ml

-5
Original file line numberDiff line numberDiff line change
@@ -106,11 +106,6 @@ let set
106106
contexts
107107
~f:(fun ((ctx : Build_context.t), ctx_type) -> ctx.name, (ctx, ctx_type)))
108108
in
109-
let () =
110-
match (cache_config : Dune_cache.Config.t) with
111-
| Disabled -> ()
112-
| Enabled _ -> Dune_cache_storage.Layout.create_cache_directories ()
113-
in
114109
Fdecl.set
115110
t
116111
{ contexts

‎src/dune_engine/clflags.ml

+1
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,4 @@ let promote = ref None
1919
let force = ref false
2020
let always_show_command_line = ref false
2121
let display = ref Display.Quiet
22+
let can_go_in_shared_cache_default = ref false

‎src/dune_engine/clflags.mli

+3
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,6 @@ val always_show_command_line : bool ref
3434

3535
(** The display mode *)
3636
val display : Display.t ref
37+
38+
(** Whether actions are cacheable by default, default [false] *)
39+
val can_go_in_shared_cache_default : bool ref

‎src/dune_rules/dune

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
build_path_prefix_map
3434
dune_engine
3535
dune_vcs
36+
dune_cache_storage
3637
dune_config
3738
dune_config_file
3839
dune_findlib

‎src/dune_rules/simple_rules.ml

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ let add_user_rule
6363
sctx
6464
~dir
6565
~(rule : Rule_conf.t)
66-
~(action : _ Action_builder.With_targets.t)
66+
~(action : Action.Full.t Action_builder.With_targets.t)
6767
~expander
6868
=
6969
let action =

‎test/blackbox-tests/test-cases/default-cache.t

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
The dune cache should be enabled by default
2+
3+
$ echo "(lang dune 3.17)" > dune-project
4+
5+
$ cat > dune << EOF
6+
> (library
7+
> (name foo))
8+
> EOF
9+
10+
$ cat > foo.ml << EOF
11+
> let f x y = x + y
12+
> EOF
13+
14+
Set up cache directory
15+
16+
$ export DUNE_CACHE_ROOT=$(pwd)/dune_test_cache
17+
$ mkdir $DUNE_CACHE_ROOT
18+
$ DUNE_CACHE=disabled dune build
19+
$ ls $DUNE_CACHE_ROOT
20+
21+
We have not written anything to the cache yet.
22+
23+
Change source files to force a recompilation
24+
25+
$ cat > foo.ml << EOF
26+
> let f x y = x - y
27+
> EOF
28+
$ dune build
29+
$ ls $DUNE_CACHE_ROOT | sort
30+
files
31+
meta
32+
temp
33+
values
34+
35+
Cache has been written to!

‎test/blackbox-tests/test-cases/directory-targets/cache-file-and-dir.t

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ This checks what happens when a file available in the cache is used in a directo
22

33
$ export DUNE_CACHE_ROOT=$PWD/.cache
44
$ export DUNE_CACHE=enabled
5+
$ export DUNE_CACHE_RULES=enabled
56
$ . ./helpers.sh
67

78
$ cat > dune-project << EOF

‎test/blackbox-tests/test-cases/directory-targets/cache-shared-subdir.t

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
We create 2 directory targets which share a whole subdirectory.
22

33
$ export DUNE_CACHE_ROOT=$PWD/.cache
4+
$ export DUNE_CACHE_RULES=enabled
45
$ export DUNE_CACHE=enabled
56
$ . ./helpers.sh
67

‎test/blackbox-tests/test-cases/directory-targets/cache.t

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
We test that directory targets can go in the shared cache. See #8067.
22

33
$ export DUNE_CACHE_ROOT=$PWD/.cache
4+
$ export DUNE_CACHE_RULES=enabled
45
$ export DUNE_CACHE=enabled
56

67
In project a, we create a rule with a directory target. The script that creates

‎test/blackbox-tests/test-cases/dune-cache/config.t

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ Check that old cache configuration format works fine with an old language
2626
Test that DUNE_CACHE_ROOT can be used to control the cache location
2727

2828
$ export DUNE_CACHE_ROOT=$PWD/.cache
29+
$ export DUNE_CACHE_RULES=enabled
2930

3031
Build succeeds and the 'copy' mode is respected
3132

‎test/blackbox-tests/test-cases/dune-cache/dedup.t

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ Test deduplication of build artifacts when using Dune cache with hard links.
22

33
$ export DUNE_CACHE=enabled
44
$ export DUNE_CACHE_ROOT=$PWD/.cache
5+
$ export DUNE_CACHE_RULES=enabled
56

67
$ cat > dune-project <<EOF
78
> (lang dune 2.1)

‎test/blackbox-tests/test-cases/dune-cache/missing-cache-entries.t

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
Check that Dune cache can cope with missing file/metadata entries.
22

33
$ export DUNE_CACHE_ROOT=$PWD/.cache
4+
$ export DUNE_CACHE_RULES=enabled
45

56
$ cat > config <<EOF
67
> (lang dune 2.1)

‎test/blackbox-tests/test-cases/dune-cache/mode-copy.t

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ variable, and via the [DUNE_CACHE_ROOT] variable. Here we test the former.
55

66
$ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime
77
$ export XDG_CACHE_HOME=$PWD/.xdg-cache
8+
$ export DUNE_CACHE_RULES=enabled
89

910
$ cat > config <<EOF
1011
> (lang dune 3.0)

‎test/blackbox-tests/test-cases/dune-cache/mode-hardlink.t

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ variable, and via the [DUNE_CACHE_ROOT] variable. Here we test the former.
55

66
$ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime
77
$ export XDG_CACHE_HOME=$PWD/.xdg-cache
8+
$ export DUNE_CACHE_RULES=enabled
89

910
$ cat > config <<EOF
1011
> (lang dune 2.1)

‎test/blackbox-tests/test-cases/dune-cache/readonly-fs.t

+15-7
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,24 @@ where Dune is supposed to store the cache:
1717
$ export DUNE_CACHE_ROOT=$(pwd)/readonly/cache-dir
1818

1919
$ dune build
20-
Error:
21-
mkdir($TESTCASE_ROOT/readonly/cache-dir): Permission denied
22-
[1]
20+
Warning: Cache directories could not be created: Permission denied; disabling
21+
cache
22+
Hint: Make sure the directory
23+
$TESTCASE_ROOT/readonly/cache-dir/temp
24+
can be created
2325

2426
Likewise, this should also happen if the location is set via XDG variables.
2527

2628
$ unset DUNE_CACHE_ROOT
2729
$ export XDG_CACHE_HOME=$(pwd)/readonly/xdg-cache-dir
30+
$ export DUNE_CONFIG__SKIP_LINE_BREAK=enabled
2831

29-
$ dune build
30-
Error:
31-
mkdir($TESTCASE_ROOT/readonly/xdg-cache-dir): Permission denied
32-
[1]
32+
$ dune build 2>&1 | sed 's/created: .*;/created: $REASON:/'
33+
Warning: Cache directories could not be created: $REASON: disabling cache
34+
Hint: Make sure the directory $TESTCASE_ROOT/readonly/xdg-cache-dir/dune/db/temp can be created
35+
36+
$ HOME=/homeless-shelter
37+
$ unset XDG_CACHE_HOME
38+
$ dune build 2>&1 | sed 's/created: .*;/created: $REASON:/'
39+
Warning: Cache directories could not be created: $REASON: disabling cache
40+
Hint: Make sure the directory /homeless-shelter/.cache/dune/db/temp can be created

‎test/blackbox-tests/test-cases/dune-cache/repro-check.t

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
Test reproducibility check
22

33
$ export DUNE_CACHE_ROOT=$PWD/.cache
4+
$ export DUNE_CACHE_RULES=enabled
45
$ cat > config <<EOF
56
> (lang dune 3.0)
67
> (cache enabled)

‎test/blackbox-tests/test-cases/dune-cache/size.t/run.t

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ the cache.
33

44
$ export DUNE_CACHE=enabled
55
$ export DUNE_CACHE_ROOT=$PWD/.cache
6+
$ export DUNE_CACHE_RULES=enabled
67

78
$ cat > config << EOF
89
> (lang dune 3.7)

‎test/blackbox-tests/test-cases/dune-cache/symlink.t

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ produced symbolic links work correctly and are appropriately cached.
33

44
$ export DUNE_CACHE=enabled
55
$ export DUNE_CACHE_ROOT=$PWD/.cache
6+
$ export DUNE_CACHE_RULES=enabled
67

78
$ cat > dune-project <<EOF
89
> (lang dune 2.1)

‎test/blackbox-tests/test-cases/dune-cache/trim.t

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
$ export DUNE_CACHE=enabled
2+
$ export DUNE_CACHE_RULES=enabled
23
$ export XDG_RUNTIME_DIR=$PWD/.xdg-runtime
34
$ export XDG_CACHE_HOME=$PWD/.xdg-cache
45

‎test/blackbox-tests/test-cases/pkg/toolchain-installation.t

+1-3
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,7 @@ but the fake compiler will end up installed as a toolchain package.
8585
Unrecognized line: "Hello from fake ocamlc!"
8686

8787
Enumerate the contents of the fake toolchains directory:
88-
$ find fake-cache | sort | remove_hash
89-
fake-cache
90-
fake-cache/dune
88+
$ find fake-cache/dune/toolchains | sort | remove_hash
9189
fake-cache/dune/toolchains
9290
fake-cache/dune/toolchains/ocaml-base-compiler.1-HASH
9391
fake-cache/dune/toolchains/ocaml-base-compiler.1-HASH/target

‎test/expect-tests/dune_config_file/dune_config_test.ml

+4-4
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ let%expect_test "cache-check-probability 0.1" =
2222
; concurrency = Fixed 1
2323
; terminal_persistence = Clear_on_rebuild
2424
; sandboxing_preference = []
25-
; cache_enabled = Disabled
25+
; cache_enabled = Enabled
2626
; cache_reproducibility_check = Check_with_probability 0.1
27-
; cache_storage_mode = None
27+
; cache_storage_mode = Some Hardlink
2828
; action_stdout_on_success = Print
2929
; action_stderr_on_success = Print
3030
; experimental = []
@@ -40,7 +40,7 @@ let%expect_test "cache-storage-mode copy" =
4040
; concurrency = Fixed 1
4141
; terminal_persistence = Clear_on_rebuild
4242
; sandboxing_preference = []
43-
; cache_enabled = Disabled
43+
; cache_enabled = Enabled
4444
; cache_reproducibility_check = Skip
4545
; cache_storage_mode = Some Copy
4646
; action_stdout_on_success = Print
@@ -58,7 +58,7 @@ let%expect_test "cache-storage-mode hardlink" =
5858
; concurrency = Fixed 1
5959
; terminal_persistence = Clear_on_rebuild
6060
; sandboxing_preference = []
61-
; cache_enabled = Disabled
61+
; cache_enabled = Enabled
6262
; cache_reproducibility_check = Skip
6363
; cache_storage_mode = Some Hardlink
6464
; action_stdout_on_success = Print

0 commit comments

Comments
 (0)