From ac8568048f6c8b328fc1c67731e0edd5daaed13f Mon Sep 17 00:00:00 2001 From: Florian Hammerschmidt Date: Fri, 25 Sep 2026 20:51:24 +0200 Subject: [PATCH 1/3] Keep compiler dependency state for the project lifetime Signed-off-by: Florian Hammerschmidt --- compiler/bsc/rescript_compiler_driver.ml | 13 +- compiler/bsc/rescript_compiler_driver.mli | 27 ++- compiler/ext/compiler_request_output.ml | 143 ++++++++++----- compiler/ml/env.ml | 144 +++++++++++---- compiler/ml/env.mli | 8 + rewatch-ocaml/README.md | 15 +- rewatch-ocaml/bench/README.md | 27 +++ rewatch-ocaml/build.ml | 19 +- rewatch-ocaml/build_attempt.ml | 8 + rewatch-ocaml/build_attempt.mli | 8 + rewatch-ocaml/build_preparation.ml | 5 +- rewatch-ocaml/build_session.ml | 10 +- rewatch-ocaml/build_session.mli | 5 + rewatch-ocaml/compiler_process.ml | 54 +++--- rewatch-ocaml/compiler_process.mli | 11 +- rewatch-ocaml/package_compilation.ml | 2 + rewatch-ocaml/package_parse.ml | 4 +- .../build_attempt_tests.ml | 18 ++ .../compiler_driver_tests.ml | 167 +++++++++++++++++- .../compiler_process_tests.ml | 57 ++++++ 20 files changed, 618 insertions(+), 127 deletions(-) diff --git a/compiler/bsc/rescript_compiler_driver.ml b/compiler/bsc/rescript_compiler_driver.ml index 703d2d6ae3..89e143d1f9 100644 --- a/compiler/bsc/rescript_compiler_driver.ml +++ b/compiler/bsc/rescript_compiler_driver.ml @@ -547,6 +547,9 @@ let () = Ident.capture_request_baseline () type result = {exit_code: int; stdout: string; stderr: string} +type session = {dependencies: Env.dependency_cache} + +let create_session () = {dependencies = Env.create_dependency_cache ()} let build_identity = Rescript_compiler_build_identity.value @@ -614,7 +617,11 @@ let run_argv ?run_external ~cwd argv = Cmt_format.set_args argv; let execute () = try - Bsc_args.parse_exn ~argv (command_line_flags ()) anonymous ~usage; + let flags = + Compiler_phase_trace.section "request.flags" command_line_flags + in + Compiler_phase_trace.section "request.dispatch" (fun () -> + Bsc_args.parse_exn ~argv flags anonymous ~usage); 0 with | Request_exit code -> code @@ -658,6 +665,10 @@ let run_request ~run_external ~cwd ~argv ~input = let logical_argv = Array.of_list ("bsc" :: (argv @ [input])) in run_argv ?run_external ~cwd logical_argv +let run_request_in_session session ~run_external ~cwd ~argv ~input = + Env.with_dependency_cache session.dependencies (fun () -> + run_request ~run_external ~cwd ~argv ~input) + let run argv = let result = run_argv ~cwd:(Sys.getcwd ()) argv in prerr_string result.stderr; diff --git a/compiler/bsc/rescript_compiler_driver.mli b/compiler/bsc/rescript_compiler_driver.mli index e84ef7f944..89e279a5be 100644 --- a/compiler/bsc/rescript_compiler_driver.mli +++ b/compiler/bsc/rescript_compiler_driver.mli @@ -1,4 +1,7 @@ type result = {exit_code: int; stdout: string; stderr: string} +type session + +val create_session : unit -> session val build_identity : string (** A digest of the compiler implementation linked into this driver. The @@ -13,14 +16,22 @@ val run_request : result (** Run one compiler request in the logical working directory. [argv] contains only options; [input] is kept separate so build-system callers cannot - accidentally construct a request without a compilation input. Requests are - serialized by the caller because the native compiler owns global mutable - state. Compiler and external-command stdout and stderr are captured in the - result, including for help, version, formatting, and reprinting requests. - Ordinary argument, parse, type, and compilation outcomes are returned as an - exit code and never terminate the host process. The request resolves file - I/O against [cwd] without changing the process working directory. Request - state is restored on success and failure. *) + accidentally construct a request without a compilation input. Each request + has fresh inference, environment, and diagnostic state. Compiler and + external-command stdout and stderr are captured in the result. Ordinary + argument, parse, type, and compilation outcomes are returned as an exit + code and never terminate the host process. File I/O resolves against [cwd] + without changing the process working directory. *) + +val run_request_in_session : + session -> + run_external:(string -> int * string * string) option -> + cwd:string -> + argv:string list -> + input:string -> + result +(** Run a module job with project-owned dependency information. Each job still + receives fresh inference and request state. *) val run : string array -> int (** Shared command-line entry point used by the standalone [bsc] wrapper. *) diff --git a/compiler/ext/compiler_request_output.ml b/compiler/ext/compiler_request_output.ml index 427161961e..f583a21993 100644 --- a/compiler/ext/compiler_request_output.ml +++ b/compiler/ext/compiler_request_output.ml @@ -1,6 +1,8 @@ +type target = {buffer: Buffer.t; mutable file: (string * out_channel) option} + type streams = { - stdout: out_channel; - stderr: out_channel; + stdout: target; + stderr: target; stdout_formatter: Format.formatter; stderr_formatter: Format.formatter; } @@ -9,14 +11,52 @@ let key = Domain.DLS.new_key (fun () -> None) let current () = Domain.DLS.get key let is_active () = Option.is_some (current ()) +let create_target () = {buffer = Buffer.create 128; file = None} + +let write_substring target text offset length = + match target.file with + | None -> Buffer.add_substring target.buffer text offset length + | Some (_, channel) -> output_substring channel text offset length + +let write target text = write_substring target text 0 (String.length text) + +let flush target = + match target.file with + | None -> () + | Some (_, channel) -> flush channel + +let make_formatter target = + Format.make_formatter (write_substring target) (fun () -> flush target) + +(* Most requests only emit text through the formatter. An out_channel is + needed for binary AST output and the few channel-based printers, so create + its temporary file only when a caller asks for one. *) +let channel target formatter = + Format.pp_print_flush formatter (); + match target.file with + | Some (_, channel) -> channel + | None -> + let path, channel = + Filename.open_temp_file ~mode:[Open_binary] "rescript-compiler-output-" + ".log" + in + (try output_string channel (Buffer.contents target.buffer) + with exn -> + close_out_noerr channel; + Sys.remove path; + raise exn); + Buffer.clear target.buffer; + target.file <- Some (path, channel); + channel + let stdout_channel () = match current () with - | Some streams -> streams.stdout + | Some streams -> channel streams.stdout streams.stdout_formatter | None -> Stdlib.stdout let stderr_channel () = match current () with - | Some streams -> streams.stderr + | Some streams -> channel streams.stderr streams.stderr_formatter | None -> Stdlib.stderr let stdout_formatter () = @@ -29,58 +69,67 @@ let stderr_formatter () = | Some streams -> streams.stderr_formatter | None -> Format.err_formatter -let write_stdout text = output_string (stdout_channel ()) text -let write_stderr text = output_string (stderr_channel ()) text -let print_stdout text = write_stdout (text ^ "\n") -let print_stderr text = write_stderr (text ^ "\n") +let write_stdout text = + match current () with + | Some streams -> write streams.stdout text + | None -> output_string Stdlib.stdout text + +let write_stderr text = + match current () with + | Some streams -> write streams.stderr text + | None -> output_string Stdlib.stderr text + +let print_stdout text = + write_stdout text; + write_stdout "\n" + +let print_stderr text = + write_stderr text; + write_stderr "\n" + +let cleanup target = + match target.file with + | None -> () + | Some (path, channel) -> ( + target.file <- None; + close_out_noerr channel; + try Sys.remove path with Sys_error _ -> ()) + +let contents target = + match target.file with + | None -> Buffer.contents target.buffer + | Some (path, channel) -> + Stdlib.flush channel; + close_out channel; + target.file <- None; + Fun.protect + (fun () -> + let input = open_in_bin path in + Fun.protect + (fun () -> really_input_string input (in_channel_length input)) + ~finally:(fun () -> close_in input)) + ~finally:(fun () -> Sys.remove path) let with_capture action = - let stdout_path, stdout = - Filename.open_temp_file ~mode:[Open_binary] "rescript-compiler-stdout-" - ".log" - in - let stderr_path, stderr = - try - Filename.open_temp_file ~mode:[Open_binary] "rescript-compiler-stderr-" - ".log" - with exn -> - close_out_noerr stdout; - Sys.remove stdout_path; - raise exn - in - let previous = current () in + let stdout = create_target () in + let stderr = create_target () in let streams = { stdout; stderr; - stdout_formatter = Format.formatter_of_out_channel stdout; - stderr_formatter = Format.formatter_of_out_channel stderr; + stdout_formatter = make_formatter stdout; + stderr_formatter = make_formatter stderr; } in + let previous = current () in Domain.DLS.set key (Some streams); - let remove path = try Sys.remove path with Sys_error _ -> () in - let read path = - let channel = open_in_bin path in - Fun.protect - (fun () -> really_input_string channel (in_channel_length channel)) - ~finally:(fun () -> close_in channel) - in Fun.protect (fun () -> - let result = - Fun.protect action ~finally:(fun () -> - Fun.protect - (fun () -> - Format.pp_print_flush streams.stdout_formatter (); - Format.pp_print_flush streams.stderr_formatter (); - flush stdout; - flush stderr) - ~finally:(fun () -> - close_out_noerr stdout; - close_out_noerr stderr; - Domain.DLS.set key previous)) - in - (result, read stdout_path, read stderr_path)) + let result = action () in + Format.pp_print_flush streams.stdout_formatter (); + Format.pp_print_flush streams.stderr_formatter (); + (result, contents stdout, contents stderr)) ~finally:(fun () -> - remove stdout_path; - remove stderr_path) + Domain.DLS.set key previous; + cleanup stdout; + cleanup stderr) diff --git a/compiler/ml/env.ml b/compiler/ml/env.ml index 0803fd1a75..f118457678 100644 --- a/compiler/ml/env.ml +++ b/compiler/ml/env.ml @@ -1639,7 +1639,8 @@ type type_snapshot = { (* Track the mutable fields reachable from the cached signature and component tables. Unsupported memo shapes make the entry ineligible for direct reuse. *) -let snapshot_type_graph graph = +let snapshot_type_graph ~raw_signature ~expanded_signature ~target_components + ~alias_components ~stages ~require_components = let seen = Physical_type_table.create 32768 in let seen_identifiers = Physical_ident_table.create 8192 in let seen_label_arrays = Physical_label_table.create 1024 in @@ -1766,13 +1767,11 @@ let snapshot_type_graph graph = visit_ident id; Option.iter visit_module_type declaration.mtd_type in - iterator.it_signature iterator graph.raw_signature; - iterator.it_signature iterator graph.expanded_signature; - List.iter visit_signature_item graph.raw_signature; - List.iter visit_signature_item graph.expanded_signature; - Array.iter visit graph.target_ids.type_nodes; - Array.iter visit graph.signature_ids.type_nodes; - Array.iter visit graph.alias_ids.type_nodes; + iterator.it_signature iterator raw_signature; + iterator.it_signature iterator expanded_signature; + List.iter visit_signature_item raw_signature; + List.iter visit_signature_item expanded_signature; + List.iter (fun stage -> Array.iter visit stage.type_nodes) stages; let capture_components = function | Some (Structure_comps components) -> let values = components.comp_values in @@ -1851,13 +1850,14 @@ let snapshot_type_graph graph = (fun module_type -> iterator.it_module_type iterator module_type) declaration.mtd_type) modtypes - | Some (Functor_comps _) | None -> unsupported := true + | Some (Functor_comps _) -> unsupported := true + | None -> if require_components then unsupported := true in - capture_components graph.target_components; - capture_components graph.alias_components; - Array.iter visit_ident graph.target_ids.identifiers; - Array.iter visit_ident graph.signature_ids.identifiers; - Array.iter visit_ident graph.alias_ids.identifiers; + capture_components target_components; + capture_components alias_components; + List.iter + (fun (stage : allocation_stage) -> Array.iter visit_ident stage.identifiers) + stages; let identifiers = Physical_ident_table.to_seq_keys seen_identifiers |> Seq.map (fun id -> (id, id.Ident.stamp, id.Ident.flags)) @@ -1879,6 +1879,14 @@ let snapshot_type_graph graph = unsupported = !unsupported; } +let snapshot_expanded_type_graph graph = + snapshot_type_graph ~raw_signature:graph.raw_signature + ~expanded_signature:graph.expanded_signature + ~target_components:graph.target_components + ~alias_components:graph.alias_components + ~stages:[graph.target_ids; graph.signature_ids; graph.alias_ids] + ~require_components:true + let type_graph_unchanged snapshot = (not snapshot.unsupported) && Array.for_all @@ -1928,10 +1936,9 @@ type expanded_snapshot_cache_entry = { let expanded_snapshot_cache_key = Domain.DLS.new_key (fun () -> ref None) let expanded_snapshot_cache () = Domain.DLS.get expanded_snapshot_cache_key -(* Only the marshaled image crosses domain boundaries. Each compiler domain - decodes its own graph, so type inference never mutates another worker's - imported types. The lock also lets one worker prepare a shared image while - other workers wait to decode it. *) +(* The marshaled image is available to every project, while a decoded graph + moves between domains only through an exclusive project cache lease. The + lock also lets one worker prepare the shared image while others wait. *) type shared_expanded_snapshot = { key: alias_key; target_filename: string; @@ -1947,7 +1954,7 @@ let shared_expanded_snapshot_lock = Mutex.create () (* Preparing a large graph costs more than one ordinary alias expansion. Wait for a second compiler request across the process so one-off edits stay cheap. - The expanded graphs themselves remain exclusive to their compiler domains. *) + An expanded graph remains exclusive to one compiler request at a time. *) let expanded_snapshot_candidates = Hashtbl.create 8 let expanded_snapshot_candidates_lock = Mutex.create () @@ -2009,21 +2016,44 @@ type cmi_cache_entry = { stats: Unix.stats; bytes: bytes; mutable cmi: Cmi_format.cmi_infos; + mutable integrity: type_snapshot option; mutable used: bool; } +type dependency_cache = { + mutex: Mutex.t; + mutable available: dependency_cache_table list; +} + +and dependency_cache_table = { + cmis: (string, cmi_cache_entry) Hashtbl.t; + expanded_snapshot: expanded_snapshot_cache_entry option ref; +} + +let create_dependency_cache () = {mutex = Mutex.create (); available = []} + let cmi_cache_key = Domain.DLS.new_key (fun () -> Hashtbl.create 2) let cmi_cache () = Domain.DLS.get cmi_cache_key -(* These two runtime interfaces are loaded by nearly every compile request. - Their decoded graphs stay private to one compiler domain. A request may - mutate them, so [finalize_cmi_cache] restores the saved image if needed. - Resolve the path on every hit to notice newly shadowing or replaced CMIs. *) +let capture_cmi_integrity cmi = + let snapshot = + Compiler_phase_trace.dependency "dependency.cmi_cache_capture" (fun () -> + snapshot_type_graph ~raw_signature:cmi.Cmi_format.cmi_sign + ~expanded_signature:[] ~target_components:None ~alias_components:None + ~stages:[] ~require_components:false) + in + if snapshot.unsupported then None else Some snapshot + +(* Each decoded interface table belongs to one request at a time. A request + may mutate its graph, so [finalize_cmi_cache] restores the saved image before + the table returns to its project. Resolve the path on every hit to notice + newly shadowing or replaced CMIs after a watch edit. *) let load_cached_cmi ~name = if (not (expanded_snapshot_enabled ())) || Domain.DLS.get preparing_expanded_snapshot - || (name <> "Stdlib" && name <> "Pervasives") + || Sys.getenv_opt "REWATCH_PROJECT_CMI_CACHE" = Some "0" + && name <> "Stdlib" && name <> "Pervasives" then None else let cache = cmi_cache () in @@ -2036,9 +2066,20 @@ let load_cached_cmi ~name = try let stats = Unix.stat resolved_filename in let bytes = Marshal.to_bytes cmi [] in - if same_file_stats (Unix.stat resolved_filename) stats then + if + Bytes.length bytes <= 64 * 1024 + && (Hashtbl.mem cache name || Hashtbl.length cache < 32) + && same_file_stats (Unix.stat resolved_filename) stats + then Hashtbl.replace cache name - {resolved_filename; stats; bytes; cmi; used = true} + { + resolved_filename; + stats; + bytes; + cmi; + integrity = capture_cmi_integrity cmi; + used = true; + } with Sys_error _ | Unix.Unix_error _ | Invalid_argument _ -> ())); Some loaded in @@ -2072,10 +2113,23 @@ let finalize_cmi_cache () = let pristine = Compiler_phase_trace.dependency "dependency.cmi_cache_verify" (fun () -> - try Bytes.equal (Marshal.to_bytes entry.cmi []) entry.bytes - with Invalid_argument _ -> false) + match entry.integrity with + | Some snapshot -> + let typed = type_graph_unchanged snapshot in + if + typed + && Sys.getenv_opt "REWATCH_PROJECT_CMI_CACHE" = Some "audit" + && not + (Bytes.equal (Marshal.to_bytes entry.cmi []) entry.bytes) + then failwith "typed CMI integrity check missed mutation"; + typed + | None -> ( + try Bytes.equal (Marshal.to_bytes entry.cmi []) entry.bytes + with Invalid_argument _ -> false)) in - if not pristine then entry.cmi <- Marshal.from_bytes entry.bytes 0)) + if not pristine then ( + entry.cmi <- Marshal.from_bytes entry.bytes 0; + entry.integrity <- capture_cmi_integrity entry.cmi))) (cmi_cache ()) let () = cached_cmi_loader := load_cached_cmi @@ -2953,7 +3007,7 @@ let prepare_expanded_snapshot_now key = Some (Compiler_phase_trace.dependency "dependency.snapshot_capture" (fun () -> - snapshot_type_graph graph)) + snapshot_expanded_type_graph graph)) else None); in_use = false; } @@ -3012,7 +3066,7 @@ let load_expanded_snapshot ~check:_ ~name = Some (Compiler_phase_trace.dependency "dependency.snapshot_capture" (fun () -> - snapshot_type_graph graph)) + snapshot_expanded_type_graph graph)) else None); graph in @@ -3090,6 +3144,34 @@ let finalize_expanded_snapshot_cache () = | None -> ()) | _ -> () +let with_dependency_cache cache action = + let table = + Mutex.lock cache.mutex; + Fun.protect + (fun () -> + match cache.available with + | table :: rest -> + cache.available <- rest; + table + | [] -> {cmis = Hashtbl.create 32; expanded_snapshot = ref None}) + ~finally:(fun () -> Mutex.unlock cache.mutex) + in + let previous_cmis = cmi_cache () in + let previous_snapshot = expanded_snapshot_cache () in + Domain.DLS.set cmi_cache_key table.cmis; + Domain.DLS.set expanded_snapshot_cache_key table.expanded_snapshot; + Fun.protect action ~finally:(fun () -> + (* Both graphs are exclusive to this request. Finalization restores + allocation IDs and any mutated nodes before another domain leases + the same table. A failed finalization discards the table. *) + Fun.protect finalize_expanded_snapshot_cache ~finally:(fun () -> + Domain.DLS.set cmi_cache_key previous_cmis; + Domain.DLS.set expanded_snapshot_cache_key previous_snapshot); + Mutex.lock cache.mutex; + Fun.protect + (fun () -> cache.available <- table :: cache.available) + ~finally:(fun () -> Mutex.unlock cache.mutex)) + let () = prepare_expanded_snapshot := prepare_expanded_snapshot_now; cached_pers_struct_loader := load_expanded_snapshot diff --git a/compiler/ml/env.mli b/compiler/ml/env.mli index 0d48df9bd7..183976fd4b 100644 --- a/compiler/ml/env.mli +++ b/compiler/ml/env.mli @@ -222,6 +222,14 @@ val finalize_expanded_snapshot_cache : unit -> unit (* Enable the per-domain expanded CMI cache for Rewatch requests. *) val with_expanded_snapshot_cache : (unit -> 'a) -> 'a + +(* Project-owned decoded CMIs and expanded signature graphs. A request leases + one table exclusively, verifies it, and returns it to the session. Finished + worker domains therefore do not discard the dependency information. *) +type dependency_cache +val create_dependency_cache : unit -> dependency_cache +val with_dependency_cache : dependency_cache -> (unit -> 'a) -> 'a + (* Keep persistent modules, imports, usage callbacks, and memoized summaries local to a compiler request. *) diff --git a/rewatch-ocaml/README.md b/rewatch-ocaml/README.md index cc8a9d961e..5e35914a8c 100644 --- a/rewatch-ocaml/README.md +++ b/rewatch-ocaml/README.md @@ -148,7 +148,20 @@ the recursive compiler source, platform-stub, C-stub, and Dune-rule inputs and shared by the embedded driver and standalone wrapper, so nested compiler changes invalidate artifacts while rewatch-only edits do not. `package_plan.ml` owns immutable per-package build inputs, `build_session.ml` -owns prepared state retained across watch rebuilds, and `build_attempt.ml` owns +owns the project graph, package configuration, compiled-artifact freshness, +and compiler dependency cache retained across watch rebuilds. The driver gives +each module job fresh inference, diagnostics, and +environment state while the project session lends each job a private table of +decoded small interfaces and an expanded signature graph. A cache hit checks +the current load path and file identity; typed graph checks detect mutations, +and changed graphs are restored from a saved image before reuse. Once a job +ends, its table returns to the session +and can be used by a later worker domain, including after a watch edit. A full +watch rebuild reconstructs the graph but retains the compiler dependency +session for the same project. Set +`REWATCH_PROJECT_CMI_CACHE=0` to limit the small-interface cache to `Stdlib` +and `Pervasives` when comparing build performance. +`build_attempt.ml` owns attempt kinds, parse outcomes, diagnostics, counters, scheduled work, and final cleanup for one build attempt. `source_dirs.ml` owns source-directory metadata projection and serialization. `process_child.ml` diff --git a/rewatch-ocaml/bench/README.md b/rewatch-ocaml/bench/README.md index d68fd26b55..c24391e5bc 100644 --- a/rewatch-ocaml/bench/README.md +++ b/rewatch-ocaml/bench/README.md @@ -641,6 +641,33 @@ of clean-build wall time on this host and fixture; it does not establish the same gain for other projects. `make test`, `make test-rewatch`, the OCaml Rewatch integration script, and the focused Rewatch OUnit suite passed. +A later project-session implementation retains tables of up to 32 decoded CMIs +of at most 64 KiB each and one expanded signature graph. A module request +borrows one table exclusively and returns it after graph verification, so a +later worker domain can reuse it across build phases and watch edits while +creating fresh inference state. On +this host, nine interleaved clean-build pairs of a synthetic 1,201-module +project measured 839.4 ms for the runtime-only cache and 843.4 ms for the +project cache after the table-lease change; that difference is within run +variation. A seven-edit retained-watch gate on a small fixture measured 77 ms +for each mode, with matching compiler work and output. The larger testrepo +gate could not run in this environment because its installed `sury-ppx` binary +reported `Exec format error`. The performance benefit of cross-edit decoded +interface and expanded-signature reuse on larger projects remains unmeasured. + +The request driver now captures ordinary text output in memory, opening a +temporary file only if a job requests an output channel for binary ASTs or +channel-based printing. The same synthetic clean build's summed parse-request +setup time fell from 1,211 ms to 11 ms across 1,202 requests; a traced build +fell from about 0.74 s to 0.62 s. Typed integrity checks on decoded CMIs +replaced repeated full serialization, reducing summed CMI verification from +about 170 ms to 8 ms across 1,201 implementation jobs. An audit mode that also +serialized the CMIs found no missed mutation on this fixture. Nine untraced +interleaved clean-build pairs with both changes gave medians of 696.2 ms for +the runtime-only CMI cache and 695.5 ms for the broader project cache, still +within run variation. These fixtures establish the request-overhead reduction +but no separate wall-time win from caching project CMIs. + One more temporary trace split the remaining WebAPI cache-hit work. Across 128 hits, forcing the cached target signature took under 1 ms, alias-ID relocation took 17 ms, and target-signature-ID relocation took 11 ms in summed worker diff --git a/rewatch-ocaml/build.ml b/rewatch-ocaml/build.ml index 5bf7e76612..00f0a3c7ec 100644 --- a/rewatch-ocaml/build.ml +++ b/rewatch-ocaml/build.ml @@ -196,8 +196,9 @@ let prepare_incremental previous changes (attempt : Build_attempt.t) |> List.map (fun source -> Compiler_process.parse_job ~bsc ~build_dir:source.package.build_dir ~config:source.package.compile_config source.source.relative_path) - |> Compiler_process.run_jobs ?poll:attempt.process_poll - ~on_complete:parse_completed + |> Compiler_process.run_jobs + ~session:(Build_session.compiler_session attempt.session) + ?poll:attempt.process_poll ~on_complete:parse_completed in let affected_modules = Hashtbl.create (List.length sources) in let dependency_updates = ref [] in @@ -319,9 +320,17 @@ let run_with_warning_state ~poll ~warning_state ~request ~no_timing ~verbosity | Some previous -> Build_attempt.create_retained ~session:previous.session ~process_poll ~progress ~verbosity - | None -> - Build_attempt.create_full ~warning_state ~process_poll ~progress - ~verbosity + | None -> ( + match request with + | Full_watch_attempt (Some previous) -> + Build_attempt.create_full_with_compiler_session + ~compiler_session:(Build_session.compiler_session previous.session) + ~warning_state ~process_poll ~progress ~verbosity + | One_shot_attempt | Initial_watch_attempt + | Full_watch_attempt None + | Retained_watch_attempt _ -> + Build_attempt.create_full ~warning_state ~process_poll ~progress + ~verbosity) in let parse_messages () = List.rev attempt.parse_messages in let parse_output messages = diff --git a/rewatch-ocaml/build_attempt.ml b/rewatch-ocaml/build_attempt.ml index da09f9bc18..a499cfce35 100644 --- a/rewatch-ocaml/build_attempt.ml +++ b/rewatch-ocaml/build_attempt.ml @@ -104,6 +104,14 @@ let create ~freshness_mode ~session ~process_poll ~progress ~verbosity = verbosity; } +let create_full_with_compiler_session ~compiler_session ~warning_state + ~process_poll ~progress ~verbosity = + create ~freshness_mode:Initialize_freshness + ~session: + (Build_session.create_with_compiler_session ~compiler_session + ~warning_state) + ~process_poll ~progress ~verbosity + let create_full ~warning_state ~process_poll ~progress ~verbosity = create ~freshness_mode:Initialize_freshness ~session:(Build_session.create ~warning_state) diff --git a/rewatch-ocaml/build_attempt.mli b/rewatch-ocaml/build_attempt.mli index 25e9746e80..aa40ec3b7f 100644 --- a/rewatch-ocaml/build_attempt.mli +++ b/rewatch-ocaml/build_attempt.mli @@ -60,6 +60,14 @@ val create_full : verbosity:int -> t +val create_full_with_compiler_session : + compiler_session:Rescript_compiler_driver.session -> + warning_state:Warning_state.t -> + process_poll:(unit -> unit) option -> + progress:Output.Progress.t -> + verbosity:int -> + t + val create_retained : session:Build_session.t -> process_poll:(unit -> unit) option -> diff --git a/rewatch-ocaml/build_preparation.ml b/rewatch-ocaml/build_preparation.ml index 586e96636d..17a64f28b8 100644 --- a/rewatch-ocaml/build_preparation.ml +++ b/rewatch-ocaml/build_preparation.ml @@ -180,8 +180,9 @@ let run ~(root_config : Config.t) ~prod ~features ~warn_error ~filter ~watch |> List.map (fun ((package : Package_plan.t), path, _) -> Compiler_process.parse_job ~bsc ~build_dir:package.build_dir ~config:package.compile_config path) - |> Compiler_process.run_jobs ?poll:attempt.process_poll - ~on_complete:parse_completed + |> Compiler_process.run_jobs + ~session:(Build_session.compiler_session attempt.session) + ?poll:attempt.process_poll ~on_complete:parse_completed in let failed_parse_paths = Hashtbl.create 8 in List.iter2 diff --git a/rewatch-ocaml/build_session.ml b/rewatch-ocaml/build_session.ml index d5ec0aa55c..cb21474f19 100644 --- a/rewatch-ocaml/build_session.ml +++ b/rewatch-ocaml/build_session.ml @@ -23,6 +23,7 @@ type cycle_cache = | Known_cycle of Module_graph.cycle_info option type t = { + compiler_session: Rescript_compiler_driver.session; global_modules: (string, Module_graph.module_node) Hashtbl.t; namespace_maps: (string, Module_graph.namespace_map) Hashtbl.t; namespace_maps_by_name: (string, Module_graph.namespace_map list) Hashtbl.t; @@ -37,8 +38,9 @@ type t = { warning_state: Warning_state.t; } -let create ~warning_state = +let create_with_compiler_session ~compiler_session ~warning_state = { + compiler_session; global_modules = Hashtbl.create 64; namespace_maps = Hashtbl.create 16; namespace_maps_by_name = Hashtbl.create 16; @@ -53,6 +55,11 @@ let create ~warning_state = warning_state; } +let create ~warning_state = + create_with_compiler_session + ~compiler_session:(Rescript_compiler_driver.create_session ()) + ~warning_state + let is_ready session = match session.readiness with | Ready _ -> true @@ -146,3 +153,4 @@ let set_public_outputs session root outputs = let iter_public_outputs session f = Hashtbl.iter f session.public_outputs let warning_state session = session.warning_state +let compiler_session session = session.compiler_session diff --git a/rewatch-ocaml/build_session.mli b/rewatch-ocaml/build_session.mli index 847532a519..9ba81d1f9a 100644 --- a/rewatch-ocaml/build_session.mli +++ b/rewatch-ocaml/build_session.mli @@ -26,6 +26,10 @@ type cycle_cache = | Known_cycle of Module_graph.cycle_info option val create : warning_state:Warning_state.t -> t +val create_with_compiler_session : + compiler_session:Rescript_compiler_driver.session -> + warning_state:Warning_state.t -> + t val is_ready : t -> bool val prepared : t -> prepared option val install_prepared : t -> prepared -> unit @@ -58,3 +62,4 @@ val iter_public_outputs : t -> (string -> (string, unit) Hashtbl.t -> unit) -> unit val warning_state : t -> Warning_state.t +val compiler_session : t -> Rescript_compiler_driver.session diff --git a/rewatch-ocaml/compiler_process.ml b/rewatch-ocaml/compiler_process.ml index 4644fe948c..e1f620e8b9 100644 --- a/rewatch-ocaml/compiler_process.ml +++ b/rewatch-ocaml/compiler_process.ml @@ -57,7 +57,7 @@ let exit_code = function | Unix.WEXITED code -> code | Unix.WSIGNALED signal | Unix.WSTOPPED signal -> 128 + signal -let run_in_process ?poll (job : Process.job) = +let run_in_process ?session ?poll (job : Process.job) = log_compiler_request job; match List.rev job.args with | [] -> @@ -70,19 +70,26 @@ let run_in_process ?poll (job : Process.job) = let result = time_compiler_request job (fun () -> Env.with_expanded_snapshot_cache (fun () -> - Rescript_compiler_driver.run_request ~cwd:job.cwd - ~argv:(List.rev reversed_argv) ~input - ~run_external: - (Some - (fun command -> - let command = Platform.shell_command command in - (* Signal handlers are process-wide; domain workers launch - PPXs without replacing the scheduler domain's handlers. *) - let result = - Process.run ?poll ~defer_signals:false ~cwd:job.cwd - command.program command.args - in - (exit_code result.status, result.stdout, result.stderr))))) + let run_external = + Some + (fun command -> + let command = Platform.shell_command command in + (* Signal handlers are process-wide; domain workers launch + PPXs without replacing the scheduler domain's handlers. *) + let result = + Process.run ?poll ~defer_signals:false ~cwd:job.cwd + command.program command.args + in + (exit_code result.status, result.stdout, result.stderr)) + in + match session with + | None -> + Rescript_compiler_driver.run_request ~cwd:job.cwd + ~argv:(List.rev reversed_argv) ~input ~run_external + | Some session -> + Rescript_compiler_driver.run_request_in_session session + ~cwd:job.cwd ~argv:(List.rev reversed_argv) ~input + ~run_external)) in { Process.status = Unix.WEXITED result.exit_code; @@ -90,24 +97,25 @@ let run_in_process ?poll (job : Process.job) = stderr = result.stderr; } -let run ?poll job = +let run ?session ?poll job = Option.iter (fun poll -> poll ()) poll; - let result = run_in_process ?poll job in + let result = run_in_process ?session ?poll job in Option.iter (fun poll -> poll ()) poll; result -let task job = +let task ?session job = let cancelled = Atomic.make false in Process.concurrent_task ~cancel:(fun () -> Atomic.set cancelled true) (fun () -> - run_in_process + run_in_process ?session ~poll:(fun () -> if Atomic.get cancelled then raise (Process.Interrupted 15)) job) -let run_jobs ?poll ?on_complete jobs = - jobs |> List.map task +let run_jobs ?session ?poll ?on_complete jobs = + jobs + |> List.map (task ?session) |> Process.run_tasks ~max_jobs:(Compiler_execution_mode.configured_count ()) ?poll ?on_complete @@ -161,8 +169,8 @@ let publish_compiler_artifacts ~artifact_dir ~ocaml_dir ~basename artifacts = with error -> raise (Compiler_scheduler.Publication_failure (error, !cmi_change)) -let namespace_task ~bsc ~runtime ~build_dir ~ocaml_dir ~entry ~package_dirty - ~force namespace modules = +let namespace_task ?session ~bsc ~runtime ~build_dir ~ocaml_dir ~entry + ~package_dirty ~force namespace modules = let mlmap = Filename.concat build_dir (namespace ^ ".mlmap") in let contents = let buffer = Buffer.create 128 in @@ -205,7 +213,7 @@ let namespace_task ~bsc ~runtime ~build_dir ~ocaml_dir ~entry ~package_dirty Compiler_scheduler. { task = - task + task ?session Process. { program = bsc; diff --git a/rewatch-ocaml/compiler_process.mli b/rewatch-ocaml/compiler_process.mli index 7a7b575ee4..3698d248c3 100644 --- a/rewatch-ocaml/compiler_process.mli +++ b/rewatch-ocaml/compiler_process.mli @@ -3,15 +3,22 @@ val build_identity : string val parse_job : bsc:string -> build_dir:string -> config:Config.t -> string -> Process.job val ast_dependencies : build_dir:string -> string -> string list -val run : ?poll:(unit -> unit) -> Process.job -> Process.result +val run : + ?session:Rescript_compiler_driver.session -> + ?poll:(unit -> unit) -> + Process.job -> + Process.result val run_jobs : + ?session:Rescript_compiler_driver.session -> ?poll:(unit -> unit) -> ?on_complete:(int -> unit) -> Process.job list -> Process.result list -val task : Process.job -> Process.task +val task : + ?session:Rescript_compiler_driver.session -> Process.job -> Process.task val namespace_task : + ?session:Rescript_compiler_driver.session -> bsc:string -> runtime:string -> build_dir:string -> diff --git a/rewatch-ocaml/package_compilation.ml b/rewatch-ocaml/package_compilation.ml index 2712cce9d6..33429ceb81 100644 --- a/rewatch-ocaml/package_compilation.ml +++ b/rewatch-ocaml/package_compilation.ml @@ -91,6 +91,7 @@ let prepare ~(package : Package_plan.t) ~(prepared : Build_session.prepared) else prepared_package.regular_common_args) module_ ~source_kind path |> Compiler_process.task + ~session:(Build_session.compiler_session attempt.session) in let record_published_outputs ~source_kind path = match source_kind with @@ -163,6 +164,7 @@ let prepare ~(package : Package_plan.t) ~(prepared : Build_session.prepared) || attempt.freshness_mode = Build_attempt.Initialize_freshness then Compiler_process.namespace_task + ~session:(Build_session.compiler_session attempt.session) ~bsc:prepared.compiler_context.bsc_path ~runtime:prepared.compiler_context.runtime_path ~build_dir ~ocaml_dir diff --git a/rewatch-ocaml/package_parse.ml b/rewatch-ocaml/package_parse.ml index f26ceea9da..e153b87d62 100644 --- a/rewatch-ocaml/package_parse.ml +++ b/rewatch-ocaml/package_parse.ml @@ -56,7 +56,9 @@ let run ~(package : Package_plan.t) ~(prepared : Build_session.prepared) Compiler_process.parse_job ~bsc:prepared.compiler_context.bsc_path ~build_dir ~config path) parse_paths_to_run - |> Compiler_process.run_jobs ?poll:attempt.process_poll) + |> Compiler_process.run_jobs + ~session:(Build_session.compiler_session attempt.session) + ?poll:attempt.process_poll) @ (dirty_parse_paths |> List.filter_map (fun path -> Hashtbl.find_opt attempt.preliminary_parses diff --git a/tests/rewatch_ounit_tests/build_attempt_tests.ml b/tests/rewatch_ounit_tests/build_attempt_tests.ml index 43ec16b62e..07ccf5baee 100644 --- a/tests/rewatch_ounit_tests/build_attempt_tests.ml +++ b/tests/rewatch_ounit_tests/build_attempt_tests.ml @@ -23,6 +23,22 @@ let retained_attempts_start_with_fresh_attempt_state _context = Build_attempt.cleanup_artifacts first; assert_equal 1 !cleanup_count +let full_rebuild_keeps_project_compiler_session _context = + let first = create_full () in + Build_session.mark_parse_pending first.session "Old.res"; + let compiler_session = Build_session.compiler_session first.session in + let second = + Build_attempt.create_full_with_compiler_session ~compiler_session + ~warning_state:(Warning_state.create ()) ~process_poll:None + ~progress:(Output.Progress.create ~enabled:false ~color:false) + ~verbosity:0 + in + assert_bool "a full rebuild retains the compiler dependency session" + (Build_session.compiler_session second.session == compiler_session); + assert_bool "a full rebuild reconstructs the build graph" + (first.session != second.session); + assert_equal [] (Build_session.pending_parse_paths second.session) + let output_inventory_survives_without_cleanup_work _context = let first = create_full () in let outputs = Hashtbl.create 1 in @@ -89,6 +105,8 @@ let tests = >::: [ "retained attempts start with fresh attempt state" >:: retained_attempts_start_with_fresh_attempt_state; + "full rebuild keeps project compiler session" + >:: full_rebuild_keeps_project_compiler_session; "output inventory survives without cleanup work" >:: output_inventory_survives_without_cleanup_work; "pending work is drained once" >:: pending_work_is_drained_once; diff --git a/tests/rewatch_ounit_tests/compiler_driver_tests.ml b/tests/rewatch_ounit_tests/compiler_driver_tests.ml index 67a86c87b7..96baca07f4 100644 --- a/tests/rewatch_ounit_tests/compiler_driver_tests.ml +++ b/tests/rewatch_ounit_tests/compiler_driver_tests.ml @@ -349,6 +349,32 @@ let output_capture_isolation_tests _context = (Compiler_request_output.stdout_channel () == Stdlib.stdout) "capture scopes restore the host stream" +let output_capture_channel_tests _context = + let (), stdout, stderr = + Compiler_request_output.with_capture (fun () -> + Compiler_request_output.write_stdout "before"; + Stdlib.Format.pp_print_string + (Compiler_request_output.stdout_formatter ()) + "formatted"; + output_value (Compiler_request_output.stdout_channel ()) 42; + Compiler_request_output.write_stdout "after"; + Compiler_request_output.write_stderr "error"; + output_string (Compiler_request_output.stderr_channel ()) " channel") + in + assert_equal ("beforeformatted" ^ Marshal.to_string 42 [] ^ "after") stdout; + assert_equal "error channel" stderr; + (try + ignore + (Compiler_request_output.with_capture (fun () -> + output_string (Compiler_request_output.stdout_channel ()) "x"; + failwith "capture failure")) + with + | Failure _ -> () + | exn -> raise exn); + check + (Compiler_request_output.stdout_channel () == Stdlib.stdout) + "a failed capture restores the host stream" + let annotation_isolation_tests _context = let left_ready = Atomic.make false in let right_ready = Atomic.make false in @@ -684,7 +710,7 @@ let type_node_id_isolation_tests _context = let write root name contents = Test_support.write_file (Filename.concat root name) contents -let run root ?(package = "driver-test") ?(extra = []) input = +let run ?session root ?(package = "driver-test") ?(extra = []) input = let argv = [ "-nostdlib"; @@ -699,9 +725,17 @@ let run root ?(package = "driver-test") ?(extra = []) input = ] @ extra in - ( argv, - Rescript_compiler_driver.run_request ~run_external:None ~cwd:root ~argv - ~input ) + let result = + match session with + | None -> + Rescript_compiler_driver.run_request ~run_external:None ~cwd:root ~argv + ~input + | Some session -> + Env.with_expanded_snapshot_cache (fun () -> + Rescript_compiler_driver.run_request_in_session session + ~run_external:None ~cwd:root ~argv ~input) + in + (argv, result) let expect_code expected result = assert_equal ~printer:string_of_int expected @@ -1069,6 +1103,36 @@ let combined_dependency_cache_tests _context = check (reused_first == reused_second) "one compiler domain reuses its verified-clean dependency graph"; + let project_cache = Env.create_dependency_cache () in + let project_loaded_type () = + Env.with_dependency_cache project_cache loaded_type + in + ignore (project_loaded_type ()); + let project_first = project_loaded_type () in + let project_second = + Domain.spawn project_loaded_type |> Domain.join + in + check + (project_first == project_second) + "a project session reuses its expanded graph on a later domain"; + let compiler_session = Rescript_compiler_driver.create_session () in + let compile_in_session () = + let _, result = + run ~session:compiler_session root ~extra:["-I"; root] + "Consumer.res" + in + expect_code 0 result + in + compile_in_session (); + Domain.spawn compile_in_session |> Domain.join; + assert_equal + ~printer:(fun _ -> "") + first_cmi + (File_util.read_file (Filename.concat root "Consumer.cmi")); + assert_equal + ~printer:(fun _ -> "") + first_cmt + (File_util.read_file (Filename.concat root "Consumer.cmt")); let other_domain_type = Domain.join (Domain.spawn loaded_type) in check (reused_second != other_domain_type) @@ -1106,7 +1170,15 @@ let combined_dependency_cache_tests _context = (match updated_on_another_domain.Types.desc with | Types.Tconstr (path, _, _) -> Path.name path = "string" | _ -> false) - "a new domain sees the updated dependency interface") + "a new domain sees the updated dependency interface"; + let updated_in_session = + Domain.spawn project_loaded_type |> Domain.join + in + check + (match updated_in_session.Types.desc with + | Types.Tconstr (path, _, _) -> Path.name path = "string" + | _ -> false) + "a project session invalidates its expanded graph after an edit") ~finally:(fun () -> (match previous_trace with | Some value -> Unix.putenv "REWATCH_TYPECHECK_TRACE" value @@ -1169,6 +1241,89 @@ let runtime_cmi_cache_tests _context = install second "string"; assert_equal "string" (load [second; first])) +let project_cmi_cache_tests _context = + Test_support.with_temp_dir "rewatch-project-cmi-cache-" (fun root -> + write root "Api.resi" "let value: int\n"; + expect_code 0 (snd (run root "Api.resi")); + let first = Env.create_dependency_cache () in + let second = Env.create_dependency_cache () in + let original_loader = !Env.Persistent_signature.load in + let loads = ref 0 in + (Env.Persistent_signature.load := + fun ~unit_name -> + if unit_name = "Api" then incr loads; + original_loader ~unit_name); + Fun.protect + ~finally:(fun () -> Env.Persistent_signature.load := original_loader) + (fun () -> + let path = + Path.Pdot + (Path.Pident (Ident.create_persistent "Api"), "value", Path.nopos) + in + let load cache ?(mutate = false) () = + Env.with_dependency_cache cache (fun () -> + Env.with_expanded_snapshot_cache (fun () -> + Fun.protect + (fun () -> + Compiler_request_state.with_fresh ~cwd:root (fun () -> + Env.with_fresh (fun () -> + (Compiler_request_state.current ()).load_path <- + [root]; + let value = Env.find_value path Env.empty in + let ty = value.Types.val_type in + if mutate then + ty.desc <- Types.Tvar (Some "changed"); + ty))) + ~finally:Env.finalize_expanded_snapshot_cache)) + in + ignore (load first ()); + assert_equal 1 !loads; + ignore (load first ()); + assert_equal ~msg:"one project reuses its loaded interface" 1 !loads; + Domain.spawn (fun () -> ignore (load first ())) |> Domain.join; + assert_equal + ~msg:"a later worker domain reuses the project's interface" 1 !loads; + ignore (load second ()); + assert_equal ~msg:"another project loads its own interface" 2 !loads; + write root "Api.res" "let value = 1\n"; + expect_code 0 (snd (run root "Api.res")); + write root "First.res" "let result = Api.value\n"; + write root "Second.res" "let result = Api.value\n"; + let compiler_session = Rescript_compiler_driver.create_session () in + let first_result = + snd + (run ~session:compiler_session root ~extra:["-I"; root] + "First.res") + in + assert_equal ~msg:first_result.stderr 0 first_result.exit_code; + let after_first_job = !loads in + let second_result = + Domain.spawn (fun () -> + snd + (run ~session:compiler_session root ~extra:["-I"; root] + "Second.res")) + |> Domain.join + in + assert_equal ~msg:second_result.stderr 0 second_result.exit_code; + assert_equal + ~msg:"module jobs in one compiler session reuse the interface" + after_first_job !loads; + ignore (load first ~mutate:true ()); + (match (load first ()).Types.desc with + | Types.Tconstr (type_path, _, _) -> + assert_equal "int" (Path.name type_path) + | _ -> assert_failure "expected the original interface type"); + write root "Api.resi" "let value: string\n"; + expect_code 0 (snd (run root "Api.resi")); + let before_update = !loads in + let updated = load first () in + match updated.Types.desc with + | Types.Tconstr (type_path, _, _) -> + assert_equal "string" (Path.name type_path); + assert_equal ~msg:"an updated interface is loaded again" + (before_update + 1) !loads + | _ -> assert_failure "expected the updated interface type")) + let concurrent_diagnostic_recovery_tests _context = Test_support.with_temp_dir "rewatch-driver-errors-" (fun root -> let first = Filename.concat root "first" in @@ -1426,6 +1581,7 @@ let tests = "env_cache_isolation" >:: env_cache_isolation_tests; "identifier_stamp_isolation" >:: identifier_stamp_isolation_tests; "output_capture_isolation" >:: output_capture_isolation_tests; + "output_capture_channel" >:: output_capture_channel_tests; "annotation_isolation" >:: annotation_isolation_tests; "backend_module_cache_isolation" >:: backend_module_cache_isolation_tests; @@ -1447,6 +1603,7 @@ let tests = >:: interface_namespace_and_load_path_tests; "combined_dependency_cache" >:: combined_dependency_cache_tests; "runtime_cmi_cache" >:: runtime_cmi_cache_tests; + "project_cmi_cache" >:: project_cmi_cache_tests; "concurrent_diagnostic_recovery" >:: concurrent_diagnostic_recovery_tests; "concurrent_jsx_diagnostic" >:: concurrent_jsx_diagnostic_tests; diff --git a/tests/rewatch_ounit_tests/compiler_process_tests.ml b/tests/rewatch_ounit_tests/compiler_process_tests.ml index 51d45d6ee8..b986fbe377 100644 --- a/tests/rewatch_ounit_tests/compiler_process_tests.ml +++ b/tests/rewatch_ounit_tests/compiler_process_tests.ml @@ -221,10 +221,67 @@ let domain_execution_test _context = with Invalid_argument _ -> true) "domain counts must honor the scheduler bound")) +let project_cache_survives_worker_batches_test _context = + Test_support.with_temp_dir "rewatch-project-worker-cache-" (fun root -> + let write name contents = + Test_support.write_file (Filename.concat root name) contents + in + write "Api.resi" "let value: int\n"; + write "Api.res" "let value = 1\n"; + write "First.res" "let result = Api.value\n"; + write "Second.res" "let result = Api.value\n"; + let job input = + Process. + { + program = ""; + cwd = root; + args = + [ + "-nostdlib"; + "-nopervasives"; + "-bs-project-root"; + root; + "-bs-package-name"; + "project-worker-cache"; + "-bs-package-output"; + "commonjs:.:.js"; + "-I"; + root; + input; + ]; + } + in + let succeeds result = check (Process.succeeded result) result.stderr in + succeeds (Compiler_process.run (job "Api.resi")); + succeeds (Compiler_process.run (job "Api.res")); + let session = + Build_session.create ~warning_state:(Warning_state.create ()) + |> Build_session.compiler_session + in + let original_loader = !Env.Persistent_signature.load in + let loads = Atomic.make 0 in + (Env.Persistent_signature.load := + fun ~unit_name -> + if unit_name = "Api" then ignore (Atomic.fetch_and_add loads 1); + original_loader ~unit_name); + Fun.protect + ~finally:(fun () -> Env.Persistent_signature.load := original_loader) + (fun () -> + let compile input = + Compiler_process.run_jobs ~session [job input] |> List.iter succeeds + in + compile "First.res"; + assert_equal 1 (Atomic.get loads); + compile "Second.res"; + assert_equal ~msg:"a new worker batch reuses the project cache" 1 + (Atomic.get loads))) + let tests = "compiler_process_tests" >::: [ "publication" >:: publication_tests; "domain_ppx_cancellation" >:: domain_ppx_cancellation_test; "domain_execution" >:: domain_execution_test; + "project_cache_survives_worker_batches" + >:: project_cache_survives_worker_batches_test; ] From b72fc5325c24a80f135bea829828b6fb45bec1b3 Mon Sep 17 00:00:00 2001 From: Florian Hammerschmidt Date: Fri, 25 Sep 2026 20:53:07 +0200 Subject: [PATCH 2/3] Document project-scoped compiler session in changelog Signed-off-by: Florian Hammerschmidt --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae40edbb74..9fad2cec27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ - Speed up OCaml rewatch builds that repeatedly open large signatures by reusing verified expanded signature graphs per compiler worker. https://github.com/rescript-lang/rescript/pull/8673 - Reuse decoded standard-library interfaces and share prepared signature images across OCaml rewatch workers for faster clean builds. https://github.com/rescript-lang/rescript/pull/8673 +- Keep imported interfaces and expanded signature graphs in a project-owned compiler session across module jobs and watch edits in OCaml rewatch. https://github.com/rescript-lang/rescript/pull/8675 +- Capture text output in memory during OCaml rewatch compiler jobs and use typed graph checks for cached interfaces to reduce clean-build overhead. https://github.com/rescript-lang/rescript/pull/8675 - Avoid running `rescript-schema-ppx` and `sury-ppx` on source files without an `@schema` annotation. https://github.com/rescript-lang/rescript/pull/8662 #### :house: Internal From 2e2baf54c4da80df4be9c86f6977bf4b75632d8d Mon Sep 17 00:00:00 2001 From: Florian Hammerschmidt Date: Fri, 25 Sep 2026 21:03:10 +0200 Subject: [PATCH 3/3] Clarify testrepo benchmark platform limitation Signed-off-by: Florian Hammerschmidt --- rewatch-ocaml/bench/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rewatch-ocaml/bench/README.md b/rewatch-ocaml/bench/README.md index c24391e5bc..ffaac33fa2 100644 --- a/rewatch-ocaml/bench/README.md +++ b/rewatch-ocaml/bench/README.md @@ -650,10 +650,10 @@ this host, nine interleaved clean-build pairs of a synthetic 1,201-module project measured 839.4 ms for the runtime-only cache and 843.4 ms for the project cache after the table-lease change; that difference is within run variation. A seven-edit retained-watch gate on a small fixture measured 77 ms -for each mode, with matching compiler work and output. The larger testrepo -gate could not run in this environment because its installed `sury-ppx` binary -reported `Exec format error`. The performance benefit of cross-edit decoded -interface and expanded-signature reuse on larger projects remains unmeasured. +for each mode, with matching compiler work and output. The available testrepo +dependencies came from a Linux container, so the larger testrepo gate was not +run on macOS. The performance benefit of cross-edit decoded interface and +expanded-signature reuse on larger projects remains unmeasured. The request driver now captures ordinary text output in memory, opening a temporary file only if a job requests an output channel for binary ASTs or