Make the project—not the module—the compiler’s lifetime boundary

CHANGELOG.md

@@ -30,6 +30,8 @@
 
 - Speed up OCaml rewatch builds that repeatedly open large signatures by reusing verified expanded signature graphs per compiler worker. https://github.com/rescript-lang/rescript/pull/8673
 - Reuse decoded standard-library interfaces and share prepared signature images across OCaml rewatch workers for faster clean builds. https://github.com/rescript-lang/rescript/pull/8673
+- Keep imported interfaces and expanded signature graphs in a project-owned compiler session across module jobs and watch edits in OCaml rewatch. https://github.com/rescript-lang/rescript/pull/8675
+- Capture text output in memory during OCaml rewatch compiler jobs and use typed graph checks for cached interfaces to reduce clean-build overhead. https://github.com/rescript-lang/rescript/pull/8675
 - Avoid running `rescript-schema-ppx` and `sury-ppx` on source files without an `@schema` annotation. https://github.com/rescript-lang/rescript/pull/8662
 
 #### :house: Internal

compiler/bsc/rescript_compiler_driver.ml

@@ -547,6 +547,9 @@ let () =
   Ident.capture_request_baseline ()
 
 type result = {exit_code: int; stdout: string; stderr: string}
+type session = {dependencies: Env.dependency_cache}
+
+let create_session () = {dependencies = Env.create_dependency_cache ()}
 
 let build_identity = Rescript_compiler_build_identity.value
 
@@ -614,7 +617,11 @@ let run_argv ?run_external ~cwd argv =
           Cmt_format.set_args argv;
           let execute () =
             try
-              Bsc_args.parse_exn ~argv (command_line_flags ()) anonymous ~usage;
+              let flags =
+                Compiler_phase_trace.section "request.flags" command_line_flags
+              in
+              Compiler_phase_trace.section "request.dispatch" (fun () ->
+                  Bsc_args.parse_exn ~argv flags anonymous ~usage);
               0
             with
             | Request_exit code -> code
@@ -658,6 +665,10 @@ let run_request ~run_external ~cwd ~argv ~input =
   let logical_argv = Array.of_list ("bsc" :: (argv @ [input])) in
   run_argv ?run_external ~cwd logical_argv
 
+let run_request_in_session session ~run_external ~cwd ~argv ~input =
+  Env.with_dependency_cache session.dependencies (fun () ->
+      run_request ~run_external ~cwd ~argv ~input)
+
 let run argv =
   let result = run_argv ~cwd:(Sys.getcwd ()) argv in
   prerr_string result.stderr;

compiler/bsc/rescript_compiler_driver.mli

@@ -1,4 +1,7 @@
 type result = {exit_code: int; stdout: string; stderr: string}
+type session
+
+val create_session : unit -> session
 
 val build_identity : string
 (** A digest of the compiler implementation linked into this driver. The
@@ -13,14 +16,22 @@ val run_request :
   result
 (** Run one compiler request in the logical working directory. [argv] contains
     only options; [input] is kept separate so build-system callers cannot
-    accidentally construct a request without a compilation input. Requests are
-    serialized by the caller because the native compiler owns global mutable
-    state. Compiler and external-command stdout and stderr are captured in the
-    result, including for help, version, formatting, and reprinting requests.
-    Ordinary argument, parse, type, and compilation outcomes are returned as an
-    exit code and never terminate the host process. The request resolves file
-    I/O against [cwd] without changing the process working directory. Request
-    state is restored on success and failure. *)
+    accidentally construct a request without a compilation input. Each request
+    has fresh inference, environment, and diagnostic state. Compiler and
+    external-command stdout and stderr are captured in the result. Ordinary
+    argument, parse, type, and compilation outcomes are returned as an exit
+    code and never terminate the host process. File I/O resolves against [cwd]
+    without changing the process working directory. *)
+
+val run_request_in_session :
+  session ->
+  run_external:(string -> int * string * string) option ->
+  cwd:string ->
+  argv:string list ->
+  input:string ->
+  result
+(** Run a module job with project-owned dependency information. Each job still
+    receives fresh inference and request state. *)
 
 val run : string array -> int
 (** Shared command-line entry point used by the standalone [bsc] wrapper. *)

compiler/ext/compiler_request_output.ml

@@ -1,6 +1,8 @@
+type target = {buffer: Buffer.t; mutable file: (string * out_channel) option}
+
 type streams = {
-  stdout: out_channel;
-  stderr: out_channel;
+  stdout: target;
+  stderr: target;
   stdout_formatter: Format.formatter;
   stderr_formatter: Format.formatter;
 }
@@ -9,14 +11,52 @@ let key = Domain.DLS.new_key (fun () -> None)
 let current () = Domain.DLS.get key
 let is_active () = Option.is_some (current ())
 
+let create_target () = {buffer = Buffer.create 128; file = None}
+
+let write_substring target text offset length =
+  match target.file with
+  | None -> Buffer.add_substring target.buffer text offset length
+  | Some (_, channel) -> output_substring channel text offset length
+
+let write target text = write_substring target text 0 (String.length text)
+
+let flush target =
+  match target.file with
+  | None -> ()
+  | Some (_, channel) -> flush channel
+
+let make_formatter target =
+  Format.make_formatter (write_substring target) (fun () -> flush target)
+
+(* Most requests only emit text through the formatter. An out_channel is
+   needed for binary AST output and the few channel-based printers, so create
+   its temporary file only when a caller asks for one. *)
+let channel target formatter =
+  Format.pp_print_flush formatter ();
+  match target.file with
+  | Some (_, channel) -> channel
+  | None ->
+    let path, channel =
+      Filename.open_temp_file ~mode:[Open_binary] "rescript-compiler-output-"
+        ".log"
+    in
+    (try output_string channel (Buffer.contents target.buffer)
+     with exn ->
+       close_out_noerr channel;
+       Sys.remove path;
+       raise exn);
+    Buffer.clear target.buffer;
+    target.file <- Some (path, channel);
+    channel
+
 let stdout_channel () =
   match current () with
-  | Some streams -> streams.stdout
+  | Some streams -> channel streams.stdout streams.stdout_formatter
   | None -> Stdlib.stdout
 
 let stderr_channel () =
   match current () with
-  | Some streams -> streams.stderr
+  | Some streams -> channel streams.stderr streams.stderr_formatter
   | None -> Stdlib.stderr
 
 let stdout_formatter () =
@@ -29,58 +69,67 @@ let stderr_formatter () =
   | Some streams -> streams.stderr_formatter
   | None -> Format.err_formatter
 
-let write_stdout text = output_string (stdout_channel ()) text
-let write_stderr text = output_string (stderr_channel ()) text
-let print_stdout text = write_stdout (text ^ "\n")
-let print_stderr text = write_stderr (text ^ "\n")
+let write_stdout text =
+  match current () with
+  | Some streams -> write streams.stdout text
+  | None -> output_string Stdlib.stdout text
+
+let write_stderr text =
+  match current () with
+  | Some streams -> write streams.stderr text
+  | None -> output_string Stdlib.stderr text
+
+let print_stdout text =
+  write_stdout text;
+  write_stdout "\n"
+
+let print_stderr text =
+  write_stderr text;
+  write_stderr "\n"
+
+let cleanup target =
+  match target.file with
+  | None -> ()
+  | Some (path, channel) -> (
+    target.file <- None;
+    close_out_noerr channel;
+    try Sys.remove path with Sys_error _ -> ())
+
+let contents target =
+  match target.file with
+  | None -> Buffer.contents target.buffer
+  | Some (path, channel) ->
+    Stdlib.flush channel;
+    close_out channel;
+    target.file <- None;
+    Fun.protect
+      (fun () ->
+        let input = open_in_bin path in
+        Fun.protect
+          (fun () -> really_input_string input (in_channel_length input))
+          ~finally:(fun () -> close_in input))
+      ~finally:(fun () -> Sys.remove path)
 
 let with_capture action =
-  let stdout_path, stdout =
-    Filename.open_temp_file ~mode:[Open_binary] "rescript-compiler-stdout-"
-      ".log"
-  in
-  let stderr_path, stderr =
-    try
-      Filename.open_temp_file ~mode:[Open_binary] "rescript-compiler-stderr-"
-        ".log"
-    with exn ->
-      close_out_noerr stdout;
-      Sys.remove stdout_path;
-      raise exn
-  in
-  let previous = current () in
+  let stdout = create_target () in
+  let stderr = create_target () in
   let streams =
     {
       stdout;
       stderr;
-      stdout_formatter = Format.formatter_of_out_channel stdout;
-      stderr_formatter = Format.formatter_of_out_channel stderr;
+      stdout_formatter = make_formatter stdout;
+      stderr_formatter = make_formatter stderr;
     }
   in
+  let previous = current () in
   Domain.DLS.set key (Some streams);
-  let remove path = try Sys.remove path with Sys_error _ -> () in
-  let read path =
-    let channel = open_in_bin path in
-    Fun.protect
-      (fun () -> really_input_string channel (in_channel_length channel))
-      ~finally:(fun () -> close_in channel)
-  in
   Fun.protect
     (fun () ->
-      let result =
-        Fun.protect action ~finally:(fun () ->
-            Fun.protect
-              (fun () ->
-                Format.pp_print_flush streams.stdout_formatter ();
-                Format.pp_print_flush streams.stderr_formatter ();
-                flush stdout;
-                flush stderr)
-              ~finally:(fun () ->
-                close_out_noerr stdout;
-                close_out_noerr stderr;
-                Domain.DLS.set key previous))
-      in
-      (result, read stdout_path, read stderr_path))
+      let result = action () in
+      Format.pp_print_flush streams.stdout_formatter ();
+      Format.pp_print_flush streams.stderr_formatter ();
+      (result, contents stdout, contents stderr))
     ~finally:(fun () ->
-      remove stdout_path;
-      remove stderr_path)
+      Domain.DLS.set key previous;
+      cleanup stdout;
+      cleanup stderr)