diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b1ad49..096cb34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,41 @@ # Changelog -## 5.0.1 (2026-09-22) +## 5.1.0 (unreleased) + +### Added + +- **Isolated sessions** - `py_session:template/1` prepares a Python + environment once (interpreter, `paths`, `imports`, `preload`, `env`, + `hash_seed`, `rlimits`) and `py_session:new/1` gives a fresh child process + per session that starts from it and shares no state with any other + session: module globals, `sys.modules`, environment, threads, working + directory. By default a session is forked from a zygote that already ran + the imports and preload, so it is ready in a few milliseconds instead of + the ~50 ms of a new interpreter plus its imports; `start => spawn` starts + a new interpreter per session (optionally from a `warm` pool) for code + that cannot be forked. A session is an isolated context: calls, callbacks, + calls back into the same session, interrupts, loops and `pass_fd` work + unchanged. `close/1` kills it; a session whose process dies answers with + the reason until closed and is never restarted. `run/5` runs one call in a + new session; `refresh/1` rebuilds the template after a deploy; `info/1` + reports zygotes, live sessions and forks. `start => reimport` runs each + call in a fresh module dictionary on a worker or owngil context instead, + the way Temporal's Python SDK isolates a workflow run: the function's + module is imported again per run, and the standard library, `imports` and + `passthrough` modules are shared. See `docs/sessions.md`. +- `clear_env` and `hash_seed` options for isolated contexts: the child sees + only the variables named in `env`, and every child built with the same + seed hashes strings and orders sets the same way. + +### Fixed + +- On an owngil context, a Python function that called `erlang.call`, where + the Erlang callback called the same context again, hung until the request + timeout. The context thread waited for the callback on its pipe while the + nested call sat in its queue. It now waits inline and serves the nested + call, as worker contexts do since 5.0.1. + + ### Fixed diff --git a/c_src/py_callback.c b/c_src/py_callback.c index 6ac4126..7b0d1bb 100644 --- a/c_src/py_callback.c +++ b/c_src/py_callback.c @@ -1657,9 +1657,14 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) { bool has_context_suspension = (tl_current_context != NULL && tl_allow_suspension && !loop_running); bool has_context_handler = (tl_current_context != NULL && tl_current_context->has_callback_handler); + /* An owngil context also has a callback handler (for its other + * threads), but a request with a caller must wait inline too: on the + * handler pipe a callback calling back into this context would queue + * behind the request that waits for it. */ bool has_context_inline = (tl_current_context != NULL && !tl_allow_suspension && tl_current_context->has_current_caller && - !has_context_handler && !loop_running); + (!has_context_handler || tl_current_context->is_subinterp) && + !loop_running); if (has_context_inline) { Py_ssize_t nargs = PyTuple_Size(args); @@ -3794,7 +3799,7 @@ static int create_erlang_module(void) { PyDict_SetItemString(log_globals, "__builtins__", builtins); /* Import erlang module into globals so the code can reference it */ - PyObject *sys_modules = PySys_GetObject("modules"); + PyObject *sys_modules = PyImport_GetModuleDict(); /* the interpreter's table, a dict even when sys.modules is replaced */ if (sys_modules != NULL) { PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang"); if (erlang_mod != NULL) { @@ -3896,7 +3901,7 @@ static int create_erlang_module(void) { PyDict_SetItemString(ext_globals, "__builtins__", builtins); /* Import erlang module into globals so the code can reference it */ - PyObject *sys_modules = PySys_GetObject("modules"); + PyObject *sys_modules = PyImport_GetModuleDict(); /* the interpreter's table, a dict even when sys.modules is replaced */ if (sys_modules != NULL) { PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang"); if (erlang_mod != NULL) { @@ -3953,7 +3958,7 @@ static int create_erlang_module(void) { PyDict_SetItemString(atom_globals, "__builtins__", builtins); /* Import erlang module into globals so the code can reference it */ - PyObject *sys_modules = PySys_GetObject("modules"); + PyObject *sys_modules = PyImport_GetModuleDict(); /* the interpreter's table, a dict even when sys.modules is replaced */ if (sys_modules != NULL) { PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang"); if (erlang_mod != NULL) { @@ -4043,7 +4048,7 @@ static int create_erlang_module(void) { PyDict_SetItemString(sd_globals, "__builtins__", builtins); /* Import erlang module into globals so the code can reference it */ - PyObject *sys_modules = PySys_GetObject("modules"); + PyObject *sys_modules = PyImport_GetModuleDict(); /* the interpreter's table, a dict even when sys.modules is replaced */ if (sys_modules != NULL) { PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang"); if (erlang_mod != NULL) { diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 54d77d6..7ff4fa5 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -3166,6 +3166,13 @@ static void *ctx_thread_main_owngil(void *arg) { ctx->request_term = req->request_data; ctx->reactor_buffer_ptr = req->reactor_buffer_ptr; ctx->local_env_ptr = req->local_env_ptr; + /* The caller serves erlang.call made by this request: the call + * waits inline and serves nested requests (ctx_call_erlang_inline), + * as on a worker context thread */ + ctx->has_current_caller = req->async_mode; + if (req->async_mode) { + ctx->current_caller = req->caller_pid; + } ctx->response_ok = false; ctx->response_term = 0; @@ -3195,6 +3202,7 @@ static void *ctx_thread_main_owngil(void *arg) { ctx->request_term = 0; ctx->reactor_buffer_ptr = NULL; ctx->local_env_ptr = NULL; + ctx->has_current_caller = false; /* Deliver result - async (message to caller) or blocking (condvar) */ if (req->async_mode) { diff --git a/docs/code-map.md b/docs/code-map.md index 9da9c9b..0677683 100644 --- a/docs/code-map.md +++ b/docs/code-map.md @@ -16,6 +16,9 @@ exercised by suites). Guides are in `docs/`, suites in `test/`. Start with | `py_context` | The API every mode answers (`call/eval/exec`, `interrupt`, `kill`, loops, `pass_fd`), the reply protocol and the pid to NIF reference table; `init/4` hands the process to `py_context_embedded` or `py_isolated` | live | context-affinity, workers, interrupts | `py_context_SUITE`, `py_context_process_SUITE`, `py_interrupt_SUITE`, `py_worker_loop_SUITE` | | `py_context_embedded` | Process body for `worker` and `owngil` mode: the receive loop, callbacks (suspension and pipe), worker loops | live | architecture, state-machines | same | | `py_isolated` | `gen_statem` driving a child process over the socket; restart policy | live | isolated | `py_isolated_*_SUITE` | +| `py_child` | Helpers shared by the processes that drive a Python child: executable lookup, socket listen/accept, frames, port env, rlimit flags | live | isolated | `py_isolated_*_SUITE` | +| `py_session` | Isolated sessions: a fresh child per session from a template (`template/1`, `new/1`, `close/1`, `run/4`, `refresh/1`), or a fresh module dictionary per run (`start => reimport`) | live | sessions | `py_session_SUITE` | +| `py_session_template`, `py_session_sup` | A template: zygotes that fork sessions (`priv/py_zygote.py`) or warm spawned sessions; exit reports to the session contexts | live | sessions | `py_session_SUITE` | | `py_context_router` | Pools and scheduler-affinity routing | live | pools, context-affinity | `py_context_router_SUITE`, `py_pool_SUITE` | | `py_context_sup`, `py_context_init` | Supervisor of contexts; starts the default pool at boot | live | pools | (through the above) | | `py_nif` | Erlang stubs and docs for every NIF | live | api-reference | all | @@ -79,8 +82,10 @@ loop, channels and servers. | `_erlang_impl/_mode.py` | Detects how Python is running (embedded, free-threaded, child) | all | | `_erlang_impl/_etf.py` | Pure-Python ETF codec with the `py_convert.c` mapping | isolated child | | `_erlang_impl/_isolated.py` | Child runtime: socket frames, reader thread, re-entrant main loop, interrupt signal, asyncio loop, the `erlang` shim | isolated child | +| `_erlang_impl/_reimport.py` | Re-import runs for `py_session` templates with `start => reimport`: a fresh module dictionary per run, swapped per thread | embedded (worker, owngil) | | `_erlang_impl/_shm.py` | `SharedMemory` and `SharedBuffer` wrappers over mmap | all | | `py_isolated_child.py` | Child launcher: rlimits, parent-death signal, cgroup join, connect | isolated child | +| `py_zygote.py` | Session template zygote: imports and preload once, forks one child per session, reports exits | session zygote | | `test_erlang_loop.py`, `test_async_task.py`, `test_channel_ref.py`, `tests/` | Python-side tests of the loop, tasks and channels | test | ## Tests (`test/`) diff --git a/docs/decisions/0009-isolated-sessions.md b/docs/decisions/0009-isolated-sessions.md new file mode 100644 index 0000000..653043f --- /dev/null +++ b/docs/decisions/0009-isolated-sessions.md @@ -0,0 +1,61 @@ +# 0009: Sessions fork from a prepared zygote + +Since 5.1.0. Code: `src/py_session.erl`, `src/py_session_template.erl`, +`priv/py_zygote.py`, `src/py_isolated.erl` (session origin), `src/py_child.erl`. + +## Situation + +Callers such as a durable-execution engine need each run of a Python +function to start from the same state and to leave nothing behind for the +next run. An isolated context keeps its interpreter between calls, and a +new one costs a cold interpreter start plus every import (about 50 ms +before the first import). Temporal's Python SDK isolates a workflow run by +re-importing its module in a fresh `sys.modules` inside a shared process; +Restate's does not isolate invocations at all and relies on the platform +(a container, a Lambda microVM). Neither gives a fresh process per run. + +## Decision + +A template prepares an interpreter once, in a zygote: a single-threaded +child that runs the imports and preload, builds the child runtime (the +`_isolated.Runtime` and the `erlang` module) without a socket, then forks +one child per session. The forked child connects that runtime to its own +socket and continues as a normal isolated child, so a session is an +isolated context (`py_isolated` with `session => true`). The zygote reports +each child's exit on its control socket; the template forwards it to the +session's context. `start => spawn` starts a normal child per session +instead, with an optional warm pool, for templates that cannot be forked. + +A session is never restarted: after its child dies it answers every +request with the reason until it is closed. It runs in its own scratch +directory, its stdio is detached from the zygote's port, and it sees only +the template's environment. + +`start => reimport` is the light variant for worker and owngil contexts: +no process per session, the function's module imported again in a fresh +module dictionary per run, swapped per thread as Temporal's workflow +sandbox does. It isolates module state only and is offered as that. + +Not chosen: a subinterpreter per session (about 13 ms, shares the process +environment, working directory, hash seed and C-extension state, and PyO3 +extensions refuse to load), CRIU (Linux only, needs privileges and PID +namespaces), and Wasm images (no native C extensions). + +## Consequences + +- A session costs a fork and a connect (a few milliseconds) instead of an + interpreter start and the imports. +- The zygote must stay single threaded: a template whose imports start a + thread is refused. On macOS, modules that load Objective-C cannot be + forked safely; `start => spawn` is the way out. +- All sessions of a template share its hash seed and its prepared state; + per-session randomness relies on Python's at-fork reseeding. +- `erlang` functions fail during preload: the runtime is not connected + until a session exists. +- A zygote that dies is rebuilt; its orphaned sessions are watched by pid + (`py_isolated` probes `kill(pid, 0)`), since nobody reports their exit. +- A re-import template replaces `sys.modules` and `builtins.__import__` in + its interpreter with per-thread stand-ins. C code that reads the + interpreter's own module table (the C `pickle`) does not see a run's + modules; the NIF's own lookups use `PyImport_GetModuleDict()` for that + reason. diff --git a/docs/decisions/overview.md b/docs/decisions/overview.md index 0d1f7eb..a6e2e3a 100644 --- a/docs/decisions/overview.md +++ b/docs/decisions/overview.md @@ -16,3 +16,4 @@ what was decided, what it costs, and where the code is. | [0006](0006-shared-memory-over-iommap.md) | Bulk data through iommap regions, handles as plain tuples | 5.0.0 | | [0007](0007-remove-legacy-execution-paths.md) | One execution path per mode; the legacy API is removed | 5.0.0 | | [0008](0008-pipe-io-rules.md) | Pipe I/O is non-blocking, deadlined and waited with poll | 3.1.0, 5.0.0 | +| [0009](0009-isolated-sessions.md) | Sessions fork from a prepared zygote | 5.1.0 | diff --git a/docs/isolated.md b/docs/isolated.md index 9705a70..009d517 100644 --- a/docs/isolated.md +++ b/docs/isolated.md @@ -24,6 +24,9 @@ memory bound. The public API is the one you already use with `worker` and | Startup | microseconds | milliseconds | ~40 ms | | Zero-copy `py_buffer`, channels, `erlang.schedule`, object refs | yes | yes | no (see Limits) | +To give every request or job its own fresh child, started from prepared +imports in a few milliseconds, use [Isolated Sessions](sessions.md). + ## Start a context ```erlang @@ -50,6 +53,8 @@ Options of `py_context:new/1` specific to this mode: | `rlimits` | `#{}` | `#{as => Bytes, cpu => Seconds, nofile => N}`, applied with `setrlimit` before any user code | | `cgroup` | none | Path of a cgroup v2 directory the child joins (limits written by you: `memory.max`, `cpu.max`, `pids.max`) | | `env` | `#{}` | Extra environment variables for the child | +| `clear_env` | `false` | When `true` the child inherits nothing from the VM's environment: it sees only `env` | +| `hash_seed` | `random` | `PYTHONHASHSEED` for the child (0 to 4294967295): the same seed gives the same `set` and `dict`-of-`str` iteration order in every child | | `paths` | `[]` | Extra `sys.path` entries (registered `py_import` paths and imports are applied too) | | `preload` | none | Code run once in the child before anything else | | `kill_after` | `1000` | Milliseconds between a soft interrupt and `SIGKILL` | diff --git a/docs/security.md b/docs/security.md index 858b718..a2af68a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -158,7 +158,9 @@ child process: ``` A crash kills only the child, `py_context:kill/1` is total, and rlimits or -cgroups bound resources. See [Isolated Contexts](isolated.md). +cgroups bound resources. See [Isolated Contexts](isolated.md). When each +request must also start from a clean state, with nothing left by the +previous one, give it its own session: see [Isolated Sessions](sessions.md). That is a boundary against Python *failing*, not against Python *reaching*. The child runs as the same user as the node, so it can read and write diff --git a/docs/sessions.md b/docs/sessions.md new file mode 100644 index 0000000..a67ee3f --- /dev/null +++ b/docs/sessions.md @@ -0,0 +1,232 @@ +# Isolated Sessions + +A session is a fresh Python process for one piece of work: a request, a +workflow step, a user's job. You prepare a template once (interpreter, +paths, imports, preload code, environment, limits), then every +`py_session:new/1` gives you a new process that starts from that template +and shares no state with any other session. Use sessions when one run must +never see what another run left behind: module globals, `sys.modules`, +environment changes, threads, files in its working directory. + +## Build a template + +```erlang +{ok, T} = py_session:template(#{ + paths => ["/srv/app"], + imports => [orders], + preload => <<"import orders\norders.load_rules()">>, + env => #{"TZ" => "UTC"}, + hash_seed => 0 +}). +``` + +The imports and preload run once, now. With the default `start => fork` +they run in a zygote process that every session is forked from, so a +session starts with them already done. + +## Open a session and use it + +```erlang +{ok, S} = py_session:new(T), +{ok, Result} = py_context:call(S, orders, handle, [Event, State]), +ok = py_session:close(S). +``` + +A session is an isolated context: `py_context:call/eval/exec`, callbacks +with `erlang.call`, calls back into the same session from a callback, +`py_context:interrupt/1`, `kill/1`, worker loops and `pass_fd/2` all work +on it. It is linked to the process that created it. `close/1` kills its +process; a session is never reused. + +## Run one call + +```erlang +{ok, Result} = py_session:run(T, orders, handle, [Event, State], + #{timeout => 5000}). +``` + +`run/5` opens a session, makes the call and closes it, also when the call +fails. + +## Choose how sessions start + +| `start` | How a session starts | Use it when | +|---|---|---| +| `fork` (default) | forked from the template's zygote: imports and preload are already done | the template can be forked | +| `spawn` | a new interpreter runs the imports and preload | a module starts a thread when imported, or loads Objective-C on macOS | +| `reimport` | no new process: the function's module is imported again in a fresh module dictionary, in a worker or owngil context | you only need fresh module state per run, and each run is one call (see below) | + +A fork copies only the thread that calls it, so a template whose imports +start a thread is refused: + +```erlang +{error, {template_failed, {init_failed, {threads, [<<"import-time-thread">>]}}}} = + py_session:template(#{imports => [module_with_a_thread]}). +``` + +With `start => spawn`, keep sessions started ahead of time so `new/1` does +not wait for an interpreter: + +```erlang +{ok, T} = py_session:template(#{start => spawn, warm => 4, imports => [orders]}). +``` + +With `start => fork`, `zygotes => N` runs N zygotes that fork in turn when +one is not enough. + +## Re-import runs in a worker or owngil context + +When a fresh module state per run is enough, and you do not need a process +boundary, run in an embedded context instead. This is how Temporal's Python +SDK isolates a workflow run: + +```erlang +{ok, T} = py_session:template(#{ + start => reimport, + mode => worker, %% or owngil + contexts => 4, %% runs are spread over these + paths => ["/srv/app"], + imports => [orders_models], %% imported once, shared by every run + passthrough => [pydantic] %% shared too, imported when first used +}), +{ok, Result} = py_session:run(T, orders, handle, [Event, State]). +``` + +Each `run/5` imports `orders` again in a new module dictionary, so its +globals start fresh and nothing a run leaves in them reaches the next one. +The standard library, `erlang`, `imports` and `passthrough` modules are +shared with the context. A run is one call: `py_session:new/1` answers +`{error, {not_supported, reimport}}`. Calls back into the same context from +a callback work. + +| | `fork` / `spawn` | `reimport` | +|---|---|---| +| Module globals of the function's module and what it imports | fresh | fresh | +| Standard library, `imports`, `passthrough` | fresh (`fork`: as prepared) | shared, their state persists | +| C extension state, environment, working directory, threads, hash seed | fresh (hash seed per template) | shared with the context | +| A call stuck in C | killed | runs on; interrupts land at the next bytecode | +| A segfault in a C extension | kills the session | kills the node | +| Memory and CPU limits | yes | no | +| Cost per run | a fork, or a warm child | an import of the function's module | +| Cost per call inside the run | a local socket round trip | none | + +Two things to know about re-import runs: + +- `sys.modules` in that interpreter becomes a mapping that shows each thread + its run's modules, and `builtins.__import__` is replaced, from the first + re-import template on. Code that checks `type(sys.modules) is dict` sees + the difference. +- C code that looks modules up in the interpreter's own table does not see a + run's modules. The C `pickle` is one: pickling an instance of a class + defined in a re-imported module fails (`KeyError` or `PicklingError`, + depending on the Python version). Put such classes in + a module listed in `imports`, or use `start => fork`. + +## What a session sees + +| | Sessions of one template | +|---|---| +| Module globals, `sys.modules`, `__main__` | fresh per session | +| `os.environ` | the template's `env` only; nothing from the VM unless `clear_env => false` | +| Working directory | an empty directory per session, removed on close | +| Threads, open files, sockets | none from another session | +| `random` state | reseeded per session | +| Hash seed (`set` and `dict` of `str` order) | the same for every session: `hash_seed`, or one chosen at build time | +| Imports and preload globals | the same starting state for every session | + +`print` in a session goes to the Erlang logger, tagged with the session's +context. + +## Bound each session + +```erlang +{ok, T} = py_session:template(#{ + imports => [orders], + rlimits => #{as => 512 * 1024 * 1024, cpu => 10, nofile => 256}, + kill_after => 1000 +}). +``` + +The limits apply to every session, as for isolated contexts (see +[Isolated Contexts](isolated.md)). A timeout on a call interrupts it; +`py_context:kill/1` ends the session's process at once. + +## When a session dies + +```erlang +{error, {child_exited, {signal, 6}}} = py_context:call(S, orders, crash, []), +{error, {child_exited, {signal, 6}}} = py_context:eval(S, <<"1">>), +ok = py_session:close(S). +``` + +A session whose process dies is not restarted: every request answers with +the reason until you close it. Other sessions and the template are not +affected. If a zygote dies, the template starts a new one; the sessions it +had forked keep running. + +## Refresh after a deploy + +```erlang +ok = py_session:refresh(T). +``` + +New sessions start from the new code; sessions already open keep what they +started with. + +## Inspect a template + +```erlang +#{start := fork, zygotes := [#{os_pid := _}], sessions := Live, forks := Total} = + py_session:info(T). +``` + +## What it costs + +Measured with `examples/bench_sessions.erl` on Apple silicon (macOS 27, +Python 3.14, 14 cores), p50 of new + first call + close. Run it on your +machine for your own figures: the cost of a fork grows with the size of +the prepared process. + +| How the session starts | Time to a used and closed session | +|---|---| +| `reimport` (one run, worker or owngil context) | 0.25 ms | +| `spawn` with a `warm` pool that keeps up | 0.6 ms | +| `fork` | 3.5 ms | +| `spawn` | 65 ms | +| a plain isolated context, for reference | 60 ms | + +| Throughput, 8 callers | | +|---|---| +| `fork`, one zygote | about 650 sessions a second (Linux: about 1,000) | +| `reimport` on four owngil contexts | about 13,500 runs a second | +| `reimport` on four worker contexts (one GIL) | about 3,500 runs a second | + +One zygote forks one session at a time; add `zygotes` when sessions are +opened faster than one zygote forks them. A call inside a session costs +what it costs in any isolated context (30 us p50, 70 us for a call that +calls back into the session), since it crosses the same socket. A call in +a re-import run stays in the process. + +`examples/bench_sessions_sdks.py` measures the isolation step of Temporal's +workflow sandbox (0.5 ms for a standard-library workflow, 4.6 ms when it +defines a pydantic model, since the module is imported again each run) and +of Restate's SDK (under a microsecond: it does not isolate invocations) on +the same workflow module. + +## Limits + +- A session is a process boundary, not a security boundary: it runs as the + node's user and can reach what that user can. See [Security](security.md). +- `erlang.call` and the other `erlang` functions work in a session, not + while the template prepares: preload code that calls Erlang fails with + "erlang is not connected". +- On macOS a module that loads Objective-C frameworks cannot be forked + safely; use `start => spawn` for it. +- All sessions of a template share one hash seed, so they order sets the + same way. `refresh/1` picks a new one when `hash_seed` is not set. + +## See also + +- [Isolated Contexts](isolated.md): the options sessions share +- [Security](security.md): what a process boundary does and does not bound +- [Decision 0009](decisions/0009-isolated-sessions.md): why sessions fork from a zygote diff --git a/docs/state-machines.md b/docs/state-machines.md index f2fee76..928531f 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -101,6 +101,8 @@ shows the current one and `sys:trace/2` prints transitions. +-- child exit / kill / socket error --> {restarting, Reason} --new child--> idle | +-- budget exhausted --> stop + | + +-- session => true --> {exited, Reason} ``` Per state: @@ -111,7 +113,8 @@ Per state: | `{busy, Id}` | postponed, unless from a process running a callback for this context (nested, dispatched) | dispatched | `{timeout, kill}` bound to `Id` once an interrupt was sent | | `looping` | `{error, loop_running}` | dispatched (`submit`, `pass_fd`, ...) | none | | `stopping_loop` | postponed | dispatched | `state_timeout` for the interrupt, then `{timeout, kill}` bound to `loop` | -| `{restarting, R}` | postponed | postponed | `state_timeout` waiting for the port's `exit_status` | +| `{restarting, R}` | postponed | postponed | `state_timeout` waiting for the port's `exit_status`; for a forked session child, a probe of its pid every 100 ms | +| `{exited, R}` | `{error, R}` | `{error, R}` | none | Transitions and their triggers: @@ -126,6 +129,12 @@ Transitions and their triggers: passed the handshake. `restart_allowed/1` counts restarts in `restart_period`; over `max_restarts` (or with `restart => false`) the process stops with `{child_exited, Reason}`. +- `{restarting, _}` to `{exited, Reason}`: a session (`session => true`, + started by `py_session`) is never restarted. Its child's exit comes from + the port, or for a child forked by a template from the template's + `{py_session_exited, OsPid, Code}`, or from the pid probe when the + template is gone. The session answers every request with `Reason` and + `kill/1` callers are answered at once; `stop` ends it. - `looping` to `stopping_loop`: `stop_loop/2` or the owner's `DOWN`. `stopping_loop` to `idle`: the `{loop_exit, R}` event. The interrupt `state_timeout` and the kill backstop escalate if the loop does not exit. diff --git a/examples/README.md b/examples/README.md index 977ac38..e040ca2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -127,6 +127,16 @@ Reactor buffer performance. escript examples/bench_reactor_buffer.erl ``` +### bench_sessions.erl +Isolated sessions: a fresh session (fork, spawn, warm pool) against a plain +isolated context, sessions per second, calls inside a session. +`bench_sessions_sdks.py` measures the isolation step of Temporal's and +Restate's Python SDKs on the same workflow module. +```bash +escript examples/bench_sessions.erl +python3 examples/bench_sessions_sdks.py # needs temporalio, restate-sdk, pydantic +``` + ### bench_resource_pool.erl Resource pool benchmark. ```bash diff --git a/examples/bench_sessions.erl b/examples/bench_sessions.erl new file mode 100644 index 0000000..5fb15bf --- /dev/null +++ b/examples/bench_sessions.erl @@ -0,0 +1,148 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +%%! -pa _build/default/lib/erlang_python/ebin + +%%% @doc Benchmark for isolated sessions (py_session). +%%% +%%% Measures, with the same workflow module for every row: +%%% 1. a fresh session (new + first call + close): fork template, spawn +%%% template, spawn template with a warm pool, plain isolated context +%%% 2. sessions per second with 1, 8 and 64 callers +%%% 3. a call and a re-entrant call (Python -> Erlang -> same session) +%%% 4. a re-import run (start => reimport) on worker and owngil contexts +%%% +%%% examples/bench_sessions_sdks.py measures what Temporal's workflow +%%% sandbox and Restate's per-invocation state cost on the same module, to +%%% compare the isolation step of each. +%%% +%%% Run with: +%%% rebar3 compile && escript examples/bench_sessions.erl + +-mode(compile). + +-define(PY, <<" +import sys, types, erlang +m = types.ModuleType('bench_sessions_wf'); sys.modules['bench_sessions_wf'] = m +exec(''' +from dataclasses import dataclass +import json, decimal, datetime + +@dataclass +class Step: + n: int + +STATE = {} + +def handle(event, state): + n = state.get('n', 0) + 1 + return {'commands': [['activity', 'charge', {'n': n}]], 'state': {'n': Step(n).n}} + +def reenter(n): + import erlang + return 0 if n == 0 else erlang.call('bench_reenter', erlang.self(), n) + 1 +''', m.__dict__) +">>). + +main(_) -> + {ok, _} = application:ensure_all_started(erlang_python), + py:register_function(bench_reenter, fun([Ctx, N]) -> + {ok, R} = py_context:call(Ctx, bench_sessions_wf, reenter, [N - 1]), R + end), + Fork = template(#{start => fork}), + Spawn = template(#{start => spawn}), + Warm = template(#{start => spawn, warm => 4}), + io:format("~n== fresh session: new + first call + close ==~n"), + row("fork", session_loop(Fork, 200)), + row("spawn", session_loop(Spawn, 20)), + row("spawn, warm pool", [begin timer:sleep(80), hd(session_loop(Warm, 1)) end + || _ <- lists:seq(1, 20)]), + row("plain isolated", [plain() || _ <- lists:seq(1, 20)]), + io:format("~n== sessions per second, fork template ==~n"), + [begin + L = throughput(Fork, C, 3), + io:format(" ~2w callers ~7.1f sessions/s p50 ~6.2f ms~n", [C, length(L) / 3, p50(L)]) + end || C <- [1, 8, 64]], + io:format("~n== calls inside a session ==~n"), + {ok, S} = py_session:new(Fork), + row("call", [us(fun() -> {ok, _} = py_context:call(S, bench_sessions_wf, handle, [[], #{}]) end) + || _ <- lists:seq(1, 2000)]), + row("re-entrant call", [us(fun() -> {ok, 1} = py_context:call(S, bench_sessions_wf, reenter, [1]) end) + || _ <- lists:seq(1, 1000)]), + py_session:close(S), + io:format("~n== re-import runs (no process per run) ==~n"), + %% the same workflow as a module file: a re-import run imports it anew + Dir = filename:join(py_child:sock_dir(), "bench_reimport"), + ok = filelib:ensure_dir(filename:join(Dir, "x")), + ok = file:write_file(filename:join(Dir, "bench_reimport_wf.py"), reimport_module()), + [begin + {ok, R} = py_session:template(#{start => reimport, mode => M, contexts => 4, + paths => [Dir]}), + row(atom_to_list(M) ++ ", one run", + [us(fun() -> {ok, _} = py_session:run(R, bench_reimport_wf, handle, [[], #{}]) end) + || _ <- lists:seq(1, 500)]), + L = reimport_throughput(R, 8, 3), + io:format(" ~-18s ~7.1f runs/s with 8 callers~n", [M, length(L) / 3]) + end || M <- [worker] ++ [owngil || py_nif:owngil_supported()]], + ok. + +reimport_module() -> + <<"from dataclasses import dataclass\nimport json, decimal, datetime\n\n" + "@dataclass\nclass Step:\n n: int\n\n" + "def handle(event, state):\n" + " n = state.get('n', 0) + 1\n" + " return {'commands': [['activity', 'charge', {'n': n}]], 'state': {'n': Step(n).n}}\n">>. + +reimport_throughput(T, Callers, Secs) -> + Self = self(), + Deadline = erlang:monotonic_time(millisecond) + Secs * 1000, + Loop = fun L(Acc) -> + case erlang:monotonic_time(millisecond) < Deadline of + true -> L([us(fun() -> {ok, _} = py_session:run(T, bench_reimport_wf, handle, [[], #{}]) end) | Acc]); + false -> Acc + end + end, + Pids = [spawn_link(fun() -> Self ! {done, self(), Loop([])} end) || _ <- lists:seq(1, Callers)], + lists:append([receive {done, P, L} -> L end || P <- Pids]). + +template(Opts) -> + {ok, T} = py_session:template(Opts#{preload => ?PY}), + T. + +session_loop(T, N) -> + [us(fun() -> + {ok, S} = py_session:new(T), + {ok, _} = py_context:call(S, bench_sessions_wf, handle, [[], #{}]), + py_session:close(S) + end) || _ <- lists:seq(1, N)]. + +plain() -> + us(fun() -> + {ok, C} = py_context:new(#{mode => isolated, preload => ?PY}), + {ok, _} = py_context:call(C, bench_sessions_wf, handle, [[], #{}]), + py_context:stop(C) + end). + +throughput(T, Callers, Secs) -> + Self = self(), + Deadline = erlang:monotonic_time(millisecond) + Secs * 1000, + Loop = fun L(Acc) -> + case erlang:monotonic_time(millisecond) < Deadline of + true -> L([hd(session_loop(T, 1)) | Acc]); + false -> Acc + end + end, + Pids = [spawn_link(fun() -> Self ! {done, self(), Loop([])} end) || _ <- lists:seq(1, Callers)], + lists:append([receive {done, P, L} -> L end || P <- Pids]). + +us(F) -> + T0 = erlang:monotonic_time(microsecond), + _ = F(), + erlang:monotonic_time(microsecond) - T0. + +row(Name, L) -> + S = lists:sort(L), + P = fun(Q) -> lists:nth(max(1, round(Q * length(S))), S) / 1000 end, + io:format(" ~-18s p50 ~8.3f ms p99 ~8.3f ms~n", [Name, P(0.5), P(0.99)]). + +p50(L) -> + lists:nth(length(L) div 2 + 1, lists:sort(L)) / 1000. diff --git a/examples/bench_sessions_sdks.py b/examples/bench_sessions_sdks.py new file mode 100644 index 0000000..5dead01 --- /dev/null +++ b/examples/bench_sessions_sdks.py @@ -0,0 +1,98 @@ +"""What the isolation step costs in Temporal and Restate, for comparison with +examples/bench_sessions.erl. + +Temporal: SandboxedWorkflowRunner.prepare_workflow builds a workflow +sandbox, the way a new workflow run gets one (a fresh sys.modules in which +the workflow module is imported again; the standard library and pydantic +are passed through). + +Restate: the per-invocation state its Python SDK creates (the VM state +machine). Restate has no isolation between invocations of one process. + +Run with: + uv venv /tmp/sdk-venv && uv pip install -p /tmp/sdk-venv temporalio restate-sdk pydantic + /tmp/sdk-venv/bin/python examples/bench_sessions_sdks.py +""" + +import asyncio +import os +import statistics +import sys +import tempfile +import time + +WORKFLOW = ''' +from dataclasses import dataclass +import json, decimal, datetime +from temporalio import workflow +%s + +@dataclass +class Step: + n: int + +@workflow.defn +class Orders: + @workflow.run + async def run(self, state: dict) -> dict: + n = state.get('n', 0) + 1 + return {'commands': [['activity', 'charge', {'n': n}]], 'state': {'n': Step(n).n}} +''' + + +def pct(xs, q): + xs = sorted(xs) + return xs[max(0, min(len(xs) - 1, round(q * len(xs)) - 1))] + + +def row(name, xs, unit='ms', scale=1e3): + print(' %-48s p50 %9.3f %s p99 %9.3f %s' + % (name, pct(xs, .5) * scale, unit, pct(xs, .99) * scale, unit)) + + +def timed(fn, n): + fn() + out = [] + for _ in range(n): + t = time.perf_counter() + fn() + out.append(time.perf_counter() - t) + return out + + +PYDANTIC_MODEL = ''' +from pydantic import BaseModel + +class Order(BaseModel): + id: str + amount: float +''' + + +async def temporal(name, extra): + from temporalio.worker import UnsandboxedWorkflowRunner + from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner + from temporalio.workflow import _Definition + d = tempfile.mkdtemp() + with open(os.path.join(d, name + '.py'), 'w') as f: + f.write(WORKFLOW % extra) + sys.path.insert(0, d) + import importlib + mod = importlib.import_module(name) + defn = _Definition.must_from_class(mod.Orders) + label = 'with a pydantic model' if extra else 'stdlib only' + row('temporal sandbox per run, ' + label, timed(lambda: SandboxedWorkflowRunner().prepare_workflow(defn), 300)) + row('temporal unsandboxed, ' + label, timed(lambda: UnsandboxedWorkflowRunner().prepare_workflow(defn), 300)) + + +def restate(): + from restate.vm import VMWrapper + headers = [('content-type', 'application/vnd.restate.invocation.v5')] + row('restate: per-invocation state', timed(lambda: VMWrapper(headers), 5000), 'us', 1e6) + + +if __name__ == '__main__': + print('\n== isolation step per run / invocation ==') + asyncio.run(temporal('bench_orders_wf', '')) + asyncio.run(temporal('bench_orders_wf_pydantic', PYDANTIC_MODEL)) + restate() diff --git a/priv/_erlang_impl/_isolated.py b/priv/_erlang_impl/_isolated.py index 04f6a97..c17b5a1 100644 --- a/priv/_erlang_impl/_isolated.py +++ b/priv/_erlang_impl/_isolated.py @@ -142,7 +142,8 @@ def _callback_error(reason): class Runtime: - """One per child process.""" + """One per child process. A session template's zygote builds it with + sock=None; each forked session sets sock before start().""" def __init__(self, sock, context_pid=None): self.sock = sock @@ -198,7 +199,14 @@ def _signal_main(self): # -- writing ----------------------------------------------------------- + def _check_connected(self): + if self.sock is None: + # Built by a session template's zygote, not connected yet + raise RuntimeError('erlang is not connected: this code runs while a ' + 'session template is prepared, before any session') + def _write_frame(self, frame_id, status, payload): + self._check_connected() body = bytes([status]) + payload data = _HEADER.pack(frame_id, len(body)) + body # A signal landing inside sendall would tear the frame and @@ -242,6 +250,7 @@ def request(self, term, timeout=None): On the main thread the wait also serves requests coming from Erlang, so nested calls work. Returns (status, value).""" + self._check_connected() if self.broken: raise PipeBroken(self.broken_reason) on_main = threading.current_thread() is self.main_thread @@ -265,6 +274,7 @@ def request(self, term, timeout=None): def request_async(self, term): """Send a status-3 request; returns an asyncio Future for the reply.""" + self._check_connected() if self.broken: raise PipeBroken(self.broken_reason) loop = asyncio.get_running_loop() diff --git a/priv/_erlang_impl/_reimport.py b/priv/_erlang_impl/_reimport.py new file mode 100644 index 0000000..6195b30 --- /dev/null +++ b/priv/_erlang_impl/_reimport.py @@ -0,0 +1,168 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Re-import runs for sessions started with `start => reimport`. + +Each run gets its own module dictionary: the modules the template passes +through (the standard library, `erlang`, and the names it lists) are shared +with the interpreter, every other module is imported again inside the run, +so module globals start fresh and nothing a run changes in them is seen by +the next one. This is the isolation of Temporal's Python workflow sandbox, +without its restrictions on time, randomness and I/O. + +The swap is per thread. `sys.modules` and `builtins.__import__` are replaced +once, in each interpreter, by stand-ins that use the running thread's module +dictionary when it has one and the interpreter's otherwise. A worker context +is one thread of the main interpreter, so several worker contexts run their +sandboxes at once without seeing each other's. + +Passthrough modules are imported by the interpreter itself, not inside the +run: C code looks modules up in the interpreter's own table and must find +them there. +""" + +import builtins +import importlib +import sys +import threading +from collections.abc import MutableMapping + +_tls = threading.local() +_install_lock = threading.Lock() +_real_import = builtins.__import__ +_real_modules = sys.modules + +_ALWAYS_SHARED = frozenset(sys.stdlib_module_names) | {'erlang', '_erlang_impl', 'py_event_loop'} + + +class _ThreadModules(MutableMapping): + """sys.modules as seen by each thread: its run's dictionary while a run + is active, the interpreter's otherwise. A mapping rather than a dict + subclass, so C code that reads sys.modules goes through these methods + instead of an empty dict storage.""" + + def _d(self): + mods = getattr(_tls, 'modules', None) + return _real_modules if mods is None else mods + + def __getitem__(self, key): + return self._d()[key] + + def __setitem__(self, key, value): + self._d()[key] = value + + def __delitem__(self, key): + del self._d()[key] + + def __contains__(self, key): + return key in self._d() + + def __iter__(self): + return iter(list(self._d())) + + def __len__(self): + return len(self._d()) + + def get(self, key, default=None): + return self._d().get(key, default) + + def copy(self): + return dict(self._d()) + + def __repr__(self): + return repr(self._d()) + + +def _install(): + global _real_modules + with _install_lock: + if not isinstance(sys.modules, _ThreadModules): + _real_modules = sys.modules + sys.modules = _ThreadModules() + builtins.__import__ = _import + + +def _import(name, globals=None, locals=None, fromlist=(), level=0): + mods = getattr(_tls, 'modules', None) + if mods is None: + return _real_import(name, globals, locals, fromlist, level) + sandbox = _tls.sandbox + if level == 0 and sandbox.shared(name) and name not in mods: + sandbox.pass_through(name, fromlist, mods) + return importlib.__import__(name, globals, locals, fromlist, level) + + +class Sandbox: + """Runs functions in fresh module dictionaries. `passthrough` names + (top-level packages) are shared with the interpreter.""" + + def __init__(self, passthrough=()): + _install() + self.passthrough = frozenset(passthrough) + + def shared(self, name): + root = name.split('.', 1)[0] + return root in _ALWAYS_SHARED or root in self.passthrough + + def pass_through(self, name, fromlist, mods): + """Import `name` in the interpreter, then share its module objects + (with its parents and loaded submodules) with the run.""" + _tls.modules = None + try: + _real_import(name, None, None, fromlist or (), 0) + for sub in fromlist or (): + full = name + '.' + sub + if full not in _real_modules: + try: + _real_import(full) + except ImportError: + pass # an attribute, not a submodule + finally: + _tls.modules = mods + parts = name.split('.') + for i in range(1, len(parts) + 1): + parent = '.'.join(parts[:i]) + if parent in _real_modules: + mods[parent] = _real_modules[parent] + prefix = name + '.' + for key, mod in list(_real_modules.items()): + if key.startswith(prefix) and key not in mods: + mods[key] = mod + + def run(self, module, func, args=(), kwargs=None): + prev = (getattr(_tls, 'modules', None), getattr(_tls, 'sandbox', None)) + _tls.sandbox = self + _tls.modules = {k: m for k, m in list(_real_modules.items()) if self.shared(k)} + try: + fn = getattr(importlib.import_module(module), func) + return fn(*args, **(kwargs or {})) + finally: + _tls.modules, _tls.sandbox = prev + + +_sandboxes = {} + + +def run(passthrough, module, func, args, kwargs): + """Entry point called by py_session:run/5 on a reimport template.""" + key = tuple(sorted(_as_text(p) for p in passthrough)) + sandbox = _sandboxes.get(key) + if sandbox is None: + sandbox = _sandboxes[key] = Sandbox(key) + return sandbox.run(_as_text(module), _as_text(func), list(args), dict(kwargs or {})) + + +def _as_text(v): + return v.decode('utf-8') if isinstance(v, (bytes, bytearray)) else str(v) diff --git a/priv/py_isolated_child.py b/priv/py_isolated_child.py index 006e074..6c0b393 100644 --- a/priv/py_isolated_child.py +++ b/priv/py_isolated_child.py @@ -183,13 +183,21 @@ def main(argv): _die('cannot connect to %s: %s' % (opts['socket'], exc)) from _erlang_impl import _isolated - from _erlang_impl._etf import Atom runtime = _isolated.Runtime(sock) _isolated.install_erlang_module(runtime) + serve(runtime, opts['rlimits'], rlimit_errors, cgroup_error) + + +def serve(runtime, limits, rlimit_errors, cgroup_error): + """Start a connected runtime, report ready (or the start-up problems) and + serve Erlang until it closes the socket. Shared by the spawned child and + the sessions forked by py_zygote.py. Never returns.""" + from _erlang_impl._etf import Atom + runtime.start() - if _AS_VIA_WATCHDOG and 'as' in opts['rlimits']: - _start_memory_watchdog(opts['rlimits']['as'], runtime) + if _AS_VIA_WATCHDOG and 'as' in limits: + _start_memory_watchdog(limits['as'], runtime) if rlimit_errors or cgroup_error: problems = [(Atom('rlimit'), Atom(k), msg) for k, msg in rlimit_errors] @@ -212,11 +220,10 @@ def main(argv): runtime.serve_forever() finally: try: - sock.close() + runtime.sock.close() except OSError: pass os._exit(0) - if __name__ == '__main__': main(sys.argv) diff --git a/priv/py_zygote.py b/priv/py_zygote.py new file mode 100644 index 0000000..d439c0e --- /dev/null +++ b/priv/py_zygote.py @@ -0,0 +1,258 @@ +# Copyright 2026 Benoit Chesneau +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Zygote of a session template: prepare an interpreter once, then fork one +fresh child per session. + +Started by src/py_session_template.erl as + + python3 py_zygote.py CONTROL_SOCKET_PATH + +The zygote stays single threaded (a fork only copies the calling thread) and +never serves Python requests itself. Frames on the control socket use the +format of the isolated child (_erlang_impl/_isolated.py): + + Erlang -> zygote {init, Paths, Imports, Preload} + {fork, SocketPath, ContextPid, #{rlimits, cgroup, cd}} + zygote -> Erlang replies to both, and the events + {ready, Info}, {exited, OsPid, ExitCode} + +The child runtime (_isolated.Runtime and the `erlang` module) is built here, +before the first fork, without a socket: user code can `import erlang` at +top level, and preload globals are the `__main__` namespace of every +session. After the fork the child connects the runtime to its own socket and +continues exactly like a spawned isolated child. +""" + +import os +import sys + + +def main(argv): + if len(argv) != 2: + sys.stderr.write('usage: py_zygote.py CONTROL_SOCKET_PATH\n') + os._exit(3) + priv = os.path.dirname(os.path.abspath(__file__)) + if priv not in sys.path: + sys.path.insert(0, priv) + + # Everything the session start-up path uses is imported now, once: + # imported after the fork it would be paid by every session. + import ctypes + import io + import resource # noqa: F401 + import selectors + import signal + import socket + import struct + import threading + import traceback + import py_isolated_child as child + from _erlang_impl import _etf, _isolated + from _erlang_impl._etf import Atom + ctypes.CDLL(None) + + child._arm_parent_death() + ctrl = child._connect(argv[1]) + runtime = _isolated.Runtime(None) + _isolated.install_erlang_module(runtime) + + header = struct.Struct('=QI') + + def send(frame_id, status, term): + body = bytes([status]) + _etf.encode(term) + ctrl.sendall(header.pack(frame_id, len(body)) + body) + + def event(term): + send(0, _isolated.STATUS_EVENT, term) + + def thread_names(): + return [t.name for t in threading.enumerate() if t is not threading.main_thread()] + + def init(term): + _, paths, imports, preload = term + for path in reversed(_isolated._as_list(paths)): + path = _isolated._as_text(path) + if path not in sys.path: + sys.path.insert(0, path) + import importlib + for name in _isolated._as_list(imports): + importlib.import_module(_isolated._as_text(name)) + code = _isolated._as_text(preload) + if code: + exec(compile(code, '', 'exec'), runtime.globals) + names = thread_names() + if names: + # A fork copies only this thread; the others' locks would be + # held forever in every session + return _isolated.STATUS_ERROR, (Atom('threads'), names) + return _isolated.STATUS_OK, Atom('ok') + + # SIGCHLD wakes the select loop through this pipe + rpipe, wpipe = os.pipe() + os.set_blocking(wpipe, False) + os.set_blocking(rpipe, False) + signal.set_wakeup_fd(wpipe) + signal.signal(signal.SIGCHLD, lambda *_: None) + + class LogStream(io.TextIOBase): + """sys.stdout / sys.stderr of a session: each line is logged by + the session's context in Erlang.""" + + def __init__(self, level): + self._level = Atom(level) + self._buf = '' + + def writable(self): + return True + + def write(self, text): + self._buf += text + while '\n' in self._buf: + line, self._buf = self._buf.split('\n', 1) + self._emit(line) + return len(text) + + def flush(self): + if self._buf: + line, self._buf = self._buf, '' + self._emit(line) + + def _emit(self, line): + try: + runtime.event((Atom('log'), self._level, line)) + except Exception: + pass + + def detach_stdio(runtime): + """fds 0-2 are the template's port pipes: a session holding them + would keep the port open after the zygote exits. Point them at + /dev/null and send Python-level output to Erlang instead.""" + null = os.open(os.devnull, os.O_RDWR) + for fd in (0, 1, 2): + os.dup2(null, fd) + os.close(null) + sys.stdin = open(os.devnull) + sys.stdout = LogStream('info') + sys.stderr = LogStream('warning') + + def session(path, context_pid, opts, frame_fds): + """Runs in the forked child; never returns.""" + try: + signal.set_wakeup_fd(-1) + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + for fd in frame_fds: + try: + os.close(fd) + except OSError: + pass + os.setsid() + limits = {_isolated._as_text(k): v + for k, v in _isolated._as_dict(opts.get(Atom('rlimits'))).items()} + rlimit_errors = child._apply_rlimits(limits) + cgroup = opts.get(Atom('cgroup')) + cgroup_error = child._join_cgroup(_isolated._as_text(cgroup) if cgroup else None) + cd = opts.get(Atom('cd')) + if cd: + os.chdir(_isolated._as_text(cd)) + try: + sock = child._connect(_isolated._as_text(path)) + except OSError as exc: + child._die('cannot connect to %s: %s' % (path, exc)) + runtime.sock = sock + runtime.context_pid = context_pid + detach_stdio(runtime) + child.serve(runtime, limits, rlimit_errors, cgroup_error) + except BaseException: + traceback.print_exc() + finally: + os._exit(0) + + def fork(term): + _, path, context_pid, opts = term + names = thread_names() + if names: + return _isolated.STATUS_ERROR, (Atom('threads'), names) + sys.stdout.flush() + sys.stderr.flush() + pid = os.fork() + if pid == 0: + session(path, context_pid, opts if isinstance(opts, dict) else {}, + [ctrl.fileno(), rpipe, wpipe, sel_fd]) + return _isolated.STATUS_OK, pid + + def reap(): + while True: + try: + pid, status = os.waitpid(-1, os.WNOHANG) + except ChildProcessError: + return + if pid == 0: + return + event((Atom('exited'), pid, os.waitstatus_to_exitcode(status))) + + def handle(frame_id, status, term): + tag = term[0] if isinstance(term, tuple) else term + try: + if tag == 'init': + reply = init(term) + elif tag == 'fork': + reply = fork(term) + else: + reply = _isolated.STATUS_ERROR, (Atom('unknown_request'), tag) + except BaseException as exc: + reply = _isolated.STATUS_ERROR, _isolated._exc_term(exc) + send(frame_id, reply[0], reply[1]) + + info = { + Atom('os_pid'): os.getpid(), + Atom('python_version'): '%d.%d.%d' % sys.version_info[:3], + Atom('executable'): sys.executable, + Atom('platform'): sys.platform, + Atom('hash_seed'): os.environ.get('PYTHONHASHSEED', 'random'), + } + event((Atom('ready'), info)) + + sel = selectors.DefaultSelector() + sel_fd = sel.fileno() if hasattr(sel, 'fileno') else -1 + sel.register(ctrl, selectors.EVENT_READ, 'ctrl') + sel.register(rpipe, selectors.EVENT_READ, 'chld') + buf = bytearray() + while True: + for key, _ in sel.select(): + if key.data == 'chld': + try: + os.read(rpipe, 4096) + except BlockingIOError: + pass + reap() + continue + data = ctrl.recv(1024 * 1024) + if not data: + # Erlang closed the template: sessions keep their own + # sockets and outlive us + os._exit(0) + buf += data + while len(buf) >= header.size: + frame_id, length = header.unpack_from(buf) + if len(buf) < header.size + length: + break + body = bytes(buf[header.size:header.size + length]) + del buf[:header.size + length] + handle(frame_id, body[0], _etf.decode(body[1:]) if len(body) > 1 else None) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/rebar.config b/rebar.config index ddae020..898a66d 100644 --- a/rebar.config +++ b/rebar.config @@ -71,6 +71,7 @@ <<"docs/asyncio.md">>, <<"docs/workers.md">>, <<"docs/isolated.md">>, + <<"docs/sessions.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -91,6 +92,7 @@ <<"docs/decisions/0006-shared-memory-over-iommap.md">>, <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, <<"docs/decisions/0008-pipe-io-rules.md">>, + <<"docs/decisions/0009-isolated-sessions.md">>, <<"docs/preload.md">>, <<"docs/owngil_internals.md">>, <<"docs/event_loop_architecture.md">> @@ -119,6 +121,7 @@ <<"docs/asyncio.md">>, <<"docs/workers.md">>, <<"docs/isolated.md">>, + <<"docs/sessions.md">>, <<"docs/reactor.md">>, <<"docs/process-bound-envs.md">>, <<"docs/security.md">>, @@ -145,7 +148,8 @@ <<"docs/decisions/0005-py-isolated-gen-statem.md">>, <<"docs/decisions/0006-shared-memory-over-iommap.md">>, <<"docs/decisions/0007-remove-legacy-execution-paths.md">>, - <<"docs/decisions/0008-pipe-io-rules.md">> + <<"docs/decisions/0008-pipe-io-rules.md">>, + <<"docs/decisions/0009-isolated-sessions.md">> ]} ]} ]}. diff --git a/src/erlang_python_sup.erl b/src/erlang_python_sup.erl index 1a5c459..dc8f192 100644 --- a/src/erlang_python_sup.erl +++ b/src/erlang_python_sup.erl @@ -128,6 +128,16 @@ init([]) -> modules => [py_context_sup] }, + %% Session templates (py_session) + SessionSupSpec = #{ + id => py_session_sup, + start => {py_session_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [py_session_sup] + }, + %% Context router initialization (starts contexts under py_context_sup) ContextRouterInitSpec = #{ id => py_context_init, @@ -179,7 +189,7 @@ init([]) -> }, Children = [CallbackSpec, ShmSpec, ThreadHandlerSpec, LoggerSpec, TracerSpec, - ContextSupSpec, ContextRouterInitSpec, + ContextSupSpec, SessionSupSpec, ContextRouterInitSpec, WorkerRegistrySpec, WorkerSupSpec, EventLoopSpec, EventLoopPoolSpec], diff --git a/src/py_child.erl b/src/py_child.erl new file mode 100644 index 0000000..7b8eba4 --- /dev/null +++ b/src/py_child.erl @@ -0,0 +1,377 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Plumbing shared by the processes that drive a Python child over a +%%% Unix socket: `py_isolated' (one context, one child) and +%%% `py_session_template' (the zygote sessions are forked from). +%%% +%%% Frames are `<>', +%%% the format of the blocking callback pipe. +%%% +%%% @private +%%% +%%% Owns: nothing; pure helpers and the listen/accept sequence. +%%% Never: keeps state between calls. +-module(py_child). + +-export([python_executable/1, + priv_dir/0, + sock_dir/0, + new_sock_path/1, + delete_file/1, + make_scratch_dir/1, + remove_tree/1, + listen/1, + accept/3, + tune_socket/1, + port_env/1, + check_env_opts/1, + rlimit_args/1, + cgroup_args/1, + frame/3, + send_frame/4, + parse_frame/1, + exit_reason/1, + code_reason/1, + os_pid_alive/1, + kill_os_pid/1, + to_bin/1, + to_list/1]). + +-include_lib("kernel/include/file.hrl"). + +-define(SOCKET_BUF, 1024 * 1024). + +%% @doc Python executable used for children: the `python' option, then the +%% `isolated_python' application env, then the interpreter matching the +%% embedded runtime, then `python3' from PATH. +-spec python_executable(map()) -> string() | {error, term()}. +python_executable(Opts) -> + Candidate = case maps:get(python, Opts, undefined) of + undefined -> + case application:get_env(erlang_python, isolated_python) of + {ok, P} -> P; + undefined -> default_python() + end; + P -> P + end, + resolve_exe(to_list(Candidate)). + +default_python() -> + case persistent_term:get({?MODULE, python}, undefined) of + undefined -> + Exe = try py:python_executable() catch _:_ -> "python3" end, + persistent_term:put({?MODULE, python}, Exe), + Exe; + Exe -> + Exe + end. + +resolve_exe(Exe) -> + case filename:pathtype(Exe) of + absolute -> + case filelib:is_file(Exe) of + true -> Exe; + false -> {error, {python_not_found, Exe}} + end; + _ -> + case os:find_executable(Exe) of + false -> {error, {python_not_found, Exe}}; + Found -> Found + end + end. + +%% @doc Absolute: a session child starts in its own directory, so a path +%% relative to the VM's cwd (code added with -pa _build/...) would not resolve. +-spec priv_dir() -> file:filename(). +priv_dir() -> + Dir = case code:priv_dir(erlang_python) of + {error, bad_name} -> + filename:join(filename:dirname(filename:dirname(code:which(?MODULE))), "priv"); + D -> + D + end, + filename:absname(Dir). + +%% @doc Private directory for the sockets of this node (mode 0700). Kept +%% under `$TMPDIR': a Unix socket path is limited to 104 bytes. Created once; +%% the path is cached so opening a child does not go through the file server. +-spec sock_dir() -> file:filename(). +sock_dir() -> + case persistent_term:get({?MODULE, sock_dir}, undefined) of + undefined -> + Base = case os:getenv("TMPDIR") of + false -> "/tmp"; + T -> T + end, + Dir = filename:join(Base, "erlang_python_" ++ os:getpid()), + ok = filelib:ensure_dir(filename:join(Dir, "x")), + _ = file:change_mode(Dir, 8#700), + persistent_term:put({?MODULE, sock_dir}, Dir), + Dir; + Dir -> + Dir + end. + +%% Files of a child (socket paths, session directories) are handled with +%% prim_file, as `raw' file access is: from the calling process, not +%% through the node's single file server, which would serialise every +%% session start and close. + +%% @doc Delete a file, ignoring errors. +-spec delete_file(file:filename()) -> ok. +delete_file(Path) -> + _ = prim_file:delete(Path), + ok. + +%% @doc A new empty directory under sock_dir/0. +-spec make_scratch_dir(string()) -> {ok, file:filename()} | {error, term()}. +make_scratch_dir(Prefix) -> + Dir = filename:join(sock_dir(), Prefix ++ integer_to_list(erlang:unique_integer([positive]))), + case prim_file:make_dir(Dir) of + ok -> + {ok, Dir}; + {error, enoent} -> + %% the socket directory was removed (a tmp cleaner): make it again + persistent_term:erase({?MODULE, sock_dir}), + _ = sock_dir(), + case prim_file:make_dir(Dir) of + ok -> {ok, Dir}; + {error, _} = Err -> Err + end; + {error, _} = Err -> + Err + end. + +%% @doc Remove a directory and everything in it, ignoring errors. +-spec remove_tree(file:filename()) -> ok. +remove_tree(Path) -> + case prim_file:read_link_info(Path) of + {ok, #file_info{type = directory}} -> + case prim_file:list_dir(Path) of + {ok, Names} -> [remove_tree(filename:join(Path, N)) || N <- Names]; + _ -> ok + end, + _ = prim_file:del_dir(Path), + ok; + {ok, _} -> + delete_file(Path); + {error, _} -> + ok + end. + +-spec new_sock_path(string()) -> file:filename(). +new_sock_path(Prefix) -> + filename:join(sock_dir(), + Prefix ++ integer_to_list(erlang:unique_integer([positive])) ++ ".sock"). + +%% @doc Listening socket at Path, for one child to connect to. +-spec listen(file:filename()) -> {ok, socket:socket()} | {error, term()}. +listen(Path) -> + delete_file(Path), + case socket:open(local, stream, default) of + {ok, L} -> + case socket:bind(L, #{family => local, path => Path}) of + ok -> + case socket:listen(L) of + ok -> {ok, L}; + {error, _} = Err -> socket:close(L), Err + end; + {error, _} = Err -> + socket:close(L), Err + end; + {error, Reason} -> + {error, {socket_open_failed, Reason}} + end. + +%% @doc Accept the child's connection while watching for its death, so a +%% child that dies before connecting (bad interpreter, missing script) is +%% reported instead of timing out. `{port, Port}' watches a spawned child and +%% keeps its output for the error; `{os_pid, Pid}' watches a forked one +%% through the `{py_session_exited, Pid, Code}' message its template sends. +-spec accept(socket:socket(), {port, port()} | {os_pid, pos_integer()}, timeout()) -> + {ok, socket:socket()} | {error, term()}. +accept(L, Watch, Timeout) -> + Deadline = erlang:monotonic_time(millisecond) + Timeout, + accept(L, Watch, Deadline, []). + +accept(L, Watch, Deadline, Out) -> + case socket:accept(L, nowait) of + {ok, S} -> + %% Output printed before connecting is still worth logging + [self() ! {Port, {data, D}} || {port, Port} <- [Watch], D <- lists:reverse(Out)], + {ok, S}; + {select, {select_info, _, Handle}} -> + Left = max(0, Deadline - erlang:monotonic_time(millisecond)), + receive + {'$socket', L, select, Handle} -> + accept(L, Watch, Deadline, Out); + {Port, {exit_status, Status}} when Watch =:= {port, Port} -> + _ = socket:cancel(L, {select_info, accept, Handle}), + {error, {child_exited_at_start, exit_reason(Status), + drain_port_output(Port, Out)}}; + {Port, {data, D}} when Watch =:= {port, Port} -> + %% Keep it here, not in the mailbox: re-sending it would + %% make this receive return at once and never time out + accept(L, Watch, Deadline, [D | Out]); + {py_session_exited, Pid, Code} when Watch =:= {os_pid, Pid} -> + _ = socket:cancel(L, {select_info, accept, Handle}), + {error, {child_exited_at_start, code_reason(Code), <<>>}} + after Left -> + _ = socket:cancel(L, {select_info, accept, Handle}), + Output = case Watch of + {port, Port} -> drain_port_output(Port, Out); + _ -> <<>> + end, + {error, {start_timeout, Output}} + end; + {error, Reason} -> + {error, {accept_failed, Reason}} + end. + +drain_port_output(Port, Acc) -> + receive + {Port, {data, D}} -> drain_port_output(Port, [D | Acc]) + after 50 -> + iolist_to_binary(lists:reverse(Acc)) + end. + +%% @doc Default Unix socket buffers are small (8 KB on macOS); large payloads +%% would cross in hundreds of wakeups. Best effort: the kernel clamps. +-spec tune_socket(socket:socket()) -> ok. +tune_socket(S) -> + _ = socket:setopt(S, {otp, rcvbuf}, ?SOCKET_BUF), + _ = socket:setopt(S, {socket, rcvbuf}, ?SOCKET_BUF), + _ = socket:setopt(S, {socket, sndbuf}, ?SOCKET_BUF), + ok. + +%% @doc The `{env, ...}' port option for a child. +%% +%% `env' adds variables to what the child inherits from the VM. With +%% `clear_env => true' nothing is inherited: every variable of the VM not +%% named in `env' is unset. `hash_seed' sets PYTHONHASHSEED; `random' (the +%% default) leaves the choice to Python. +-spec port_env(map()) -> [{string(), string() | false}]. +port_env(Opts) -> + Env = [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))], + Seed = case maps:get(hash_seed, Opts, random) of + random -> []; + N -> [{"PYTHONHASHSEED", integer_to_list(N)}] + end, + Set = Env ++ Seed, + Clear = case maps:get(clear_env, Opts, false) of + true -> + Keep = [K || {K, _} <- Set], + lists:usort([{K, false} || KV <- os:getenv(), + K <- [hd(string:split(KV, "="))], + K =/= "", not lists:member(K, Keep)]); + false -> + [] + end, + Clear ++ Set. + +%% @doc Check the options port_env/1 reads. +-spec check_env_opts(map()) -> ok | {error, term()}. +check_env_opts(Opts) -> + case {maps:get(hash_seed, Opts, random), maps:get(clear_env, Opts, false)} of + {Seed, _} when Seed =/= random, + not (is_integer(Seed) andalso Seed >= 0 andalso Seed =< 4294967295) -> + {error, {badarg, {hash_seed, Seed}}}; + {_, Clear} when not is_boolean(Clear) -> + {error, {badarg, {clear_env, Clear}}}; + _ -> + ok + end. + +-spec rlimit_args(map()) -> [string()]. +rlimit_args(Opts) -> + Limits = maps:get(rlimits, Opts, #{}), + lists:append([case maps:get(K, Limits, undefined) of + undefined -> []; + V when is_integer(V), V >= 0 -> ["--rlimit-" ++ atom_to_list(K), integer_to_list(V)] + end || K <- [as, cpu, nofile]]). + +-spec cgroup_args(map()) -> [string()]. +cgroup_args(Opts) -> + case maps:get(cgroup, Opts, undefined) of + undefined -> []; + Dir -> ["--cgroup", to_list(Dir)] + end. + +-spec frame(non_neg_integer(), byte(), binary()) -> binary(). +frame(Id, Status, Payload) -> + Body = <>, + <>. + +-spec send_frame(socket:socket(), non_neg_integer(), byte(), term()) -> ok | {error, term()}. +send_frame(S, Id, Status, Term) -> + case socket:send(S, frame(Id, Status, term_to_binary(Term))) of + ok -> ok; + {error, {Reason, _Rest}} -> {error, Reason}; + {error, Reason} -> {error, Reason} + end. + +-spec parse_frame(binary()) -> + {ok, {non_neg_integer(), byte(), term()}, binary()} | more | {error, term()}. +parse_frame(<>) -> + case Body of + <> -> + try + Term = case Payload of + <<>> -> undefined; + _ -> binary_to_term(Payload) + end, + {ok, {Id, Status, Term}, Rest} + catch + error:badarg -> {error, bad_etf} + end; + <<>> -> + {error, empty_body} + end; +parse_frame(_) -> + more. + +%% @doc Exit reason from a port exit status (128 + N is signal N). +-spec exit_reason(integer()) -> {signal, pos_integer()} | {exit_status, integer()}. +exit_reason(Status) when Status > 128 -> {signal, Status - 128}; +exit_reason(Status) -> {exit_status, Status}. + +%% @doc Exit reason from an `os.waitstatus_to_exitcode' value (-N is +%% signal N), as the zygote reports it. +-spec code_reason(integer()) -> {signal, pos_integer()} | {exit_status, integer()}. +code_reason(Code) when Code < 0 -> {signal, -Code}; +code_reason(Code) -> {exit_status, Code}. + +-spec kill_os_pid(integer()) -> ok. +kill_os_pid(OsPid) when is_integer(OsPid), OsPid > 0 -> + _ = py_nif:os_kill(OsPid, 9), + ok; +kill_os_pid(_) -> + ok. + +%% @doc Whether a process still exists (a zombie counts until reaped). +-spec os_pid_alive(integer()) -> boolean(). +os_pid_alive(OsPid) when is_integer(OsPid), OsPid > 0 -> + py_nif:os_kill(OsPid, 0) =/= {error, esrch}; +os_pid_alive(_) -> + false. + +to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +to_bin(L) when is_list(L) -> unicode:characters_to_binary(L); +to_bin(B) when is_binary(B) -> B. + +to_list(A) when is_atom(A) -> atom_to_list(A); +to_list(B) when is_binary(B) -> unicode:characters_to_list(B); +to_list(I) when is_integer(I) -> integer_to_list(I); +to_list(L) when is_list(L) -> L. diff --git a/src/py_isolated.erl b/src/py_isolated.erl index 566c4f8..b2b0f93 100644 --- a/src/py_isolated.erl +++ b/src/py_isolated.erl @@ -44,8 +44,20 @@ %%%
  • `{restarting, Reason}' - the child is gone or being killed; %%% requests are postponed until the new child is up. In-flight %%% requests fail with `{error, Reason}'.
    • +%%%
  • `{exited, Reason}' - a session's child is gone. Sessions are never +%%% restarted: every request answers `{error, Reason}' until the +%%% context is stopped.
    • %%% %%% +%%% == Sessions == +%%% +%%% `py_session' starts contexts with `session => true' and an `origin'. +%%% `{fork, Template}' asks a `py_session_template' to fork the child from +%%% its zygote: there is no port, the template reports the exit with +%%% `{py_session_exited, OsPid, Code}', and the child already holds the +%%% template's imports and preload. `spawn' starts one as usual. Either way +%%% the child runs in a fresh scratch directory removed when it stops. +%%% %%% The message protocol with py_context is unchanged: requests are plain %%% messages `{call, From, MRef, ...}' answered with `From ! {MRef, Reply}'. %%% Use `sys:get_state/1' to see the state and `sys:trace/2' for events. @@ -81,10 +93,12 @@ -define(DEFAULT_RESTART_PERIOD_MS, 10000). -define(SHUTDOWN_GRACE_MS, 1000). -define(EXIT_STATUS_WAIT_MS, 5000). --define(SOCKET_BUF, 1024 * 1024). +-define(EXIT_PROBE_MS, 100). +-define(EXIT_REPORT_GRACE_MS, 2000). -record(child, { - port :: port(), + %% undefined for a child forked by a session template + port :: port() | undefined, os_pid :: pos_integer(), listener :: socket:socket() | undefined, sock :: socket:socket(), @@ -119,14 +133,16 @@ %% Request id (or `loop') the armed kill backstop is bound to kill_target :: pos_integer() | loop | undefined, %% Callers of kill/1 answered once the new child is up - kill_waiters = [] :: [{pid(), reference()}] + kill_waiters = [] :: [{pid(), reference()}], + %% Session scratch directory (the child's cwd), removed on stop + scratch :: file:filename() | undefined }). -define(IS_MAIN(K), (K =:= call orelse K =:= eval orelse K =:= exec orelse (is_tuple(K) andalso element(1, K) =:= start_loop))). -type state() :: idle | {busy, pos_integer()} | looping | stopping_loop - | {restarting, term()}. + | {restarting, term()} | {exited, term()}. -export_type([state/0]). %% ============================================================================ @@ -177,6 +193,10 @@ init(_Args) -> handle_event(enter, _Old, idle, #data{kill_waiters = Waiters} = Data) -> [W ! {M, ok} || {W, M} <- Waiters], {keep_state, Data#data{kill_waiters = []}}; +handle_event(enter, _Old, {restarting, _}, #data{child = #child{port = undefined}}) -> + %% A forked child's exit comes from its template, which may be gone: + %% check the pid too + {keep_state_and_data, [{state_timeout, ?EXIT_PROBE_MS, {probe_exit, 0}}]}; handle_event(enter, _Old, {restarting, _}, _Data) -> %% SIGKILL was sent (or the child is exiting): the port reports it %% within milliseconds; this is the safety net @@ -198,9 +218,38 @@ handle_event(info, {Port, {data, Out}}, _State, #data{child = #child{port = Port log_output(Data, Out), keep_state_and_data; handle_event(info, {Port, {exit_status, Status}}, State, #data{child = #child{port = Port}} = Data) -> - child_exited(exit_reason(Status), State, Data); + child_exited(py_child:exit_reason(Status), State, Data); handle_event(info, {Port, _}, _State, _Data) when is_port(Port) -> keep_state_and_data; +handle_event(info, {py_session_exited, OsPid, Code}, State, + #data{child = #child{port = undefined, os_pid = OsPid}} = Data) -> + child_exited(py_child:code_reason(Code), State, Data); +handle_event(info, {py_session_exited, _, _}, _State, _Data) -> + keep_state_and_data; +handle_event(state_timeout, {probe_exit, Waited}, {restarting, _} = State, + #data{child = #child{os_pid = OsPid}, opts = Opts} = Data) -> + case py_child:os_pid_alive(OsPid) of + false -> + %% Reaped. Its template reports how it died, a little later + %% than the pid disappears when the machine is busy: wait for + %% that report unless nobody is left to send it. + ReportExpected = case maps:get(origin, Opts, spawn) of + {fork, T} -> is_process_alive(T) andalso Waited < ?EXIT_REPORT_GRACE_MS; + _ -> false + end, + case ReportExpected of + true -> + {keep_state_and_data, + [{state_timeout, ?EXIT_PROBE_MS, {probe_exit, Waited + ?EXIT_PROBE_MS}}]}; + false -> + child_exited({signal, 9}, State, Data) + end; + true when Waited >= ?EXIT_STATUS_WAIT_MS -> + handle_event(state_timeout, exit_status, State, Data); + true -> + {keep_state_and_data, + [{state_timeout, ?EXIT_PROBE_MS, {probe_exit, Waited + ?EXIT_PROBE_MS}}]} + end; handle_event(state_timeout, exit_status, {restarting, _} = State, #data{child = Child} = Data) -> logger:error("py_context ~p (isolated): child ~p did not exit after SIGKILL", [Data#data.id, Child#child.os_pid]), @@ -299,6 +348,9 @@ handle_event({timeout, kill}, Target, State, #data{pending = Pending} = Data) -> false -> {keep_state, Data#data{kill_target = undefined}} end; +handle_event(info, {kill, From, MRef}, {exited, _}, _Data) -> + From ! {MRef, ok}, + keep_state_and_data; handle_event(info, {kill, From, MRef}, State, Data) -> case State of {restarting, _} -> @@ -307,11 +359,30 @@ handle_event(info, {kill, From, MRef}, State, Data) -> _ -> kill(killed, State, Data#data{kill_waiters = [{From, MRef} | Data#data.kill_waiters]}) end; -handle_event(info, {stop, From, MRef}, _State, Data) -> - Data1 = stop_child(Data, graceful), +handle_event(info, {stop, From, MRef}, _State, #data{opts = Opts} = Data) -> + How = case maps:get(session, Opts, false) of + %% The zygote (or the VM, for a spawned one) reaps the child: + %% nothing to wait for once it has SIGKILL + true -> kill_nowait; + false -> graceful + end, + Data1 = stop_child(Data, How), + %% The scratch directory is removed by terminate/3, right after the + %% reply: the caller does not wait for the file system From ! {MRef, ok}, {stop, normal, Data1}; +%% A session started ahead of time by its template is handed to the process +%% that asked for it: that process becomes the parent (linked, its crash +%% stops the session) and the template lets go. +handle_event(info, {set_parent, From, MRef, NewParent}, _State, #data{parent = Old} = Data) -> + link(NewParent), + _ = Old =/= NewParent andalso unlink(Old), + %% An EXIT from the old parent may already be queued: forget it + receive {'EXIT', Old, _} -> ok after 0 -> ok end, + From ! {MRef, ok}, + {keep_state, Data#data{parent = NewParent}}; + %% ---- worker loop ----------------------------------------------------------- handle_event(info, {stop_loop, From, MRef, GraceMs}, looping, #data{loop = Loop} = Data) -> @@ -386,9 +457,14 @@ handle_event(info, _Other, _State, _Data) -> terminate(Reason, _State, #data{child = Child} = Data) -> _ = case Child of undefined -> Data; - _ when Reason =:= normal; Reason =:= shutdown -> stop_child(Data, graceful); + _ when Reason =:= normal; Reason =:= shutdown -> + case maps:get(session, Data#data.opts, false) of + true -> stop_child(Data, kill); + false -> stop_child(Data, graceful) + end; _ -> stop_child(Data, kill) end, + remove_scratch(Data), ets:delete(?REF_TAB, self()), ok. @@ -401,44 +477,11 @@ format_status(#{data := #data{child = #child{} = Child} = Data} = Status) -> format_status(Status) -> Status. -%% @doc Python executable used for isolated children: the `python' option, -%% then the `isolated_python' application env, then the interpreter matching -%% the embedded runtime, then `python3' from PATH. +%% @doc Python executable used for isolated children; see +%% py_child:python_executable/1. -spec python_executable(map()) -> string() | {error, term()}. python_executable(Opts) -> - Candidate = case maps:get(python, Opts, undefined) of - undefined -> - case application:get_env(erlang_python, isolated_python) of - {ok, P} -> P; - undefined -> default_python() - end; - P -> P - end, - resolve_exe(to_list(Candidate)). - -default_python() -> - case persistent_term:get({?MODULE, python}, undefined) of - undefined -> - Exe = try py:python_executable() catch _:_ -> "python3" end, - persistent_term:put({?MODULE, python}, Exe), - Exe; - Exe -> - Exe - end. - -resolve_exe(Exe) -> - case filename:pathtype(Exe) of - absolute -> - case filelib:is_file(Exe) of - true -> Exe; - false -> {error, {python_not_found, Exe}} - end; - _ -> - case os:find_executable(Exe) of - false -> {error, {python_not_found, Exe}}; - Found -> Found - end - end. + py_child:python_executable(Opts). %% ============================================================================ %% Child startup @@ -446,8 +489,13 @@ resolve_exe(Exe) -> start_child(#data{opts = Opts} = St) -> case check_platform_opts(Opts) of - ok -> start_child_1(St); - {error, _} = Err -> Err + ok -> + case py_child:check_env_opts(Opts) of + ok -> start_child_1(St); + {error, _} = Err -> Err + end; + {error, _} = Err -> + Err end. %% cgroups exist only on Linux; rlimits are POSIX and apply everywhere. @@ -460,106 +508,113 @@ check_platform_opts(Opts) -> {_, {unix, Os}} -> {error, {cgroup_unsupported, Os}} end. -start_child_1(#data{opts = Opts} = St) -> - case python_executable(Opts) of - {error, _} = Err -> - Err; - Python -> - case spawn_child(Python, Opts) of - {ok, Child} -> - handshake(St#data{child = Child}); - {error, _} = Err -> - Err +start_child_1(#data{opts = Opts} = St0) -> + St = with_scratch(St0), + ChildOpts = case St#data.scratch of + undefined -> Opts; + Dir -> Opts#{cd => Dir} + end, + Started = case maps:get(origin, Opts, spawn) of + {fork, Template} -> + fork_child(Template, ChildOpts); + spawn -> + case python_executable(Opts) of + {error, _} = NoPython -> NoPython; + Python -> spawn_child(Python, ChildOpts) end + end, + case Started of + {ok, Child} -> + handshake(St#data{child = Child}); + {error, _} = Failed -> + remove_scratch(St), + Failed + end. + +%% A session runs in its own empty directory, created here and removed +%% when the context stops. +with_scratch(#data{scratch = undefined, opts = #{session := true}} = St) -> + {ok, Dir} = py_child:make_scratch_dir("sess_"), + St#data{scratch = Dir}; +with_scratch(St) -> + St. + +remove_scratch(#data{scratch = undefined}) -> + ok; +remove_scratch(#data{scratch = Dir}) -> + py_child:remove_tree(Dir), + ok. + +%% Ask the session template to fork a child from its zygote; the child +%% connects to our socket like a spawned one. +fork_child(Template, Opts) -> + Path = py_child:new_sock_path("sess_"), + case py_child:listen(Path) of + {ok, L} -> + Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), + ForkOpts = maps:with([rlimits, cgroup, cd], Opts), + case py_session_template:fork(Template, Path, ForkOpts, Timeout) of + {ok, OsPid} -> + case py_child:accept(L, {os_pid, OsPid}, Timeout) of + {ok, S} -> + py_child:delete_file(Path), + py_child:tune_socket(S), + {ok, #child{port = undefined, os_pid = OsPid, listener = L, + sock = S, sock_path = Path}}; + {error, Reason} -> + py_child:delete_file(Path), + socket:close(L), + py_child:kill_os_pid(OsPid), + {error, Reason} + end; + {error, Reason} -> + py_child:delete_file(Path), + socket:close(L), + {error, Reason} + end; + {error, Reason} -> + {error, {spawn_failed, Reason}} end. spawn_child(Python, Opts) -> - Dir = sock_dir(), - Path = filename:join(Dir, "ctx_" ++ integer_to_list(erlang:unique_integer([positive])) ++ ".sock"), - _ = file:delete(Path), - case socket:open(local, stream, default) of + Path = py_child:new_sock_path("ctx_"), + case py_child:listen(Path) of {ok, L} -> try - ok = socket:bind(L, #{family => local, path => Path}), - ok = socket:listen(L), - Script = filename:join(priv_dir(), "py_isolated_child.py"), - Args = [Script, Path | rlimit_args(Opts) ++ cgroup_args(Opts)], + Script = filename:join(py_child:priv_dir(), "py_isolated_child.py"), + Args = [Script, Path | py_child:rlimit_args(Opts) ++ py_child:cgroup_args(Opts)], PortOpts = [exit_status, stderr_to_stdout, binary, use_stdio, - {args, Args}, {env, env_opt(Opts)}], + {args, Args}, {env, py_child:port_env(Opts)}] + ++ [{cd, Dir} || Dir <- [maps:get(cd, Opts, undefined)], + Dir =/= undefined], Port = open_port({spawn_executable, Python}, PortOpts), OsPid = case erlang:port_info(Port, os_pid) of {os_pid, Pid} -> Pid; _ -> 0 end, Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), - case accept_child(L, Port, Timeout) of + case py_child:accept(L, {port, Port}, Timeout) of {ok, S} -> - _ = file:delete(Path), - tune_socket(S), + py_child:delete_file(Path), + py_child:tune_socket(S), {ok, #child{port = Port, os_pid = OsPid, listener = L, sock = S, sock_path = Path}}; {error, Reason} -> - _ = file:delete(Path), + py_child:delete_file(Path), socket:close(L), kill_port(Port, OsPid), {error, Reason} end catch Class:Err:Stack -> - _ = file:delete(Path), + py_child:delete_file(Path), socket:close(L), {error, {spawn_failed, {Class, Err, Stack}}} end; + {error, {socket_open_failed, _}} = Err -> + Err; {error, Reason} -> - {error, {socket_open_failed, Reason}} - end. - -%% Accept while also watching the port: a child that dies before connecting -%% (bad interpreter, missing script) is reported with its output. -accept_child(L, Port, Timeout) -> - Deadline = erlang:monotonic_time(millisecond) + Timeout, - accept_child(L, Port, Deadline, []). - -accept_child(L, Port, Deadline, Out) -> - case socket:accept(L, nowait) of - {ok, S} -> - %% Output printed before connecting is still worth logging - [self() ! {Port, {data, D}} || D <- lists:reverse(Out)], - {ok, S}; - {select, {select_info, _, Handle}} -> - Left = max(0, Deadline - erlang:monotonic_time(millisecond)), - receive - {'$socket', L, select, Handle} -> - accept_child(L, Port, Deadline, Out); - {Port, {exit_status, Status}} -> - _ = socket:cancel(L, {select_info, accept, Handle}), - {error, {child_exited_at_start, exit_reason(Status), - drain_port_output(Port, Out)}}; - {Port, {data, D}} -> - %% Keep it here, not in the mailbox: re-sending it would - %% make this receive return at once and never time out - accept_child(L, Port, Deadline, [D | Out]) - after Left -> - _ = socket:cancel(L, {select_info, accept, Handle}), - {error, {start_timeout, drain_port_output(Port, Out)}} - end; - {error, Reason} -> - {error, {accept_failed, Reason}} - end. - -%% Default Unix socket buffers are small (8 KB on macOS); large payloads -%% would cross in hundreds of wakeups. Best effort: the kernel clamps. -tune_socket(S) -> - _ = socket:setopt(S, {otp, rcvbuf}, ?SOCKET_BUF), - _ = socket:setopt(S, {socket, rcvbuf}, ?SOCKET_BUF), - _ = socket:setopt(S, {socket, sndbuf}, ?SOCKET_BUF), - ok. - -drain_port_output(Port, Acc) -> - receive - {Port, {data, D}} -> drain_port_output(Port, [D | Acc]) - after 50 -> - iolist_to_binary(lists:reverse(Acc)) + {error, {spawn_failed, Reason}} end. %% Blocking handshake: ready event, init request, then the preload exec. @@ -567,12 +622,16 @@ handshake(#data{child = Child, opts = Opts} = St0) -> St = St0#data{}, Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), case recv_frame_sync(Child, Timeout) of + {ok, {0, ?STATUS_EVENT, {ready, Info}}, Child1} when Child1#child.port =:= undefined -> + %% Forked: imports and preload ran in the zygote, and the child + %% learnt our pid from the fork request + {ok, St#data{child = Child1#child{info = Info}}}; {ok, {0, ?STATUS_EVENT, {ready, Info}}, Child1} -> St1 = St#data{child = Child1#child{info = Info}}, - Paths = [to_bin(P) || P <- py_import:all_paths()] ++ extra_paths(Opts), + Paths = [py_child:to_bin(P) || P <- py_import:all_paths()] ++ extra_paths(Opts), %% Registered imports are pre-cached in sys.modules, as %% interp_apply_imports does for the embedded modes - Imports = lists:usort([to_bin(M) || {M, _} <- py_import:all_imports()]), + Imports = lists:usort([py_child:to_bin(M) || {M, _} <- py_import:all_imports()]), case sync_request(St1, {init, self(), Paths, Imports}, Timeout) of {{ok, _}, St2} -> run_preload(St2, Timeout); @@ -624,7 +683,7 @@ py_preload_code() -> end. extra_paths(Opts) -> - [to_bin(P) || P <- maps:get(paths, Opts, [])]. + [py_child:to_bin(P) || P <- maps:get(paths, Opts, [])]. %% Send a request and wait for its reply, ignoring nothing: callbacks made %% by the child during startup are served too. @@ -673,7 +732,7 @@ sync_wait(#data{child = Child} = St, Id, Timeout) -> end. recv_frame_sync(#child{buf = Buf} = Child, Timeout) -> - case parse_frame(Buf) of + case py_child:parse_frame(Buf) of {ok, Frame, Rest} -> {ok, Frame, Child#child{buf = Rest}}; more -> @@ -703,6 +762,9 @@ request(From, MRef, {start_loop, _}, _Term, State, _Data) request(From, MRef, Kind, _Term, looping, _Data) when ?IS_MAIN(Kind) -> From ! {MRef, {error, loop_running}}, keep_state_and_data; +request(From, MRef, _Kind, _Term, {exited, Reason}, _Data) -> + From ! {MRef, {error, Reason}}, + keep_state_and_data; request(_From, _MRef, _Kind, _Term, {restarting, _}, _Data) -> {keep_state_and_data, [postpone]}; request(_From, _MRef, Kind, _Term, stopping_loop, _Data) when ?IS_MAIN(Kind) -> @@ -735,11 +797,14 @@ dispatch(From, MRef, Kind, Term, State, #data{child = Child, next_id = Id, pendi socket_broken(Reason, State, Data, []) end. +pass_fd(From, MRef, _Fd, {exited, Reason}, _Data) -> + From ! {MRef, {error, Reason}}, + keep_state_and_data; pass_fd(_From, _MRef, _Fd, {restarting, _}, _Data) -> {keep_state_and_data, [postpone]}; pass_fd(From, MRef, Fd, _State, #data{child = Child, next_id = Id, pending = Pending} = Data) when is_integer(Fd), Fd >= 0 -> - Frame = frame(Id, ?STATUS_REQUEST, term_to_binary(pass_fd)), + Frame = py_child:frame(Id, ?STATUS_REQUEST, term_to_binary(pass_fd)), Msg = #{iov => [Frame], ctrl => [#{level => socket, type => rights, data => <>}]}, case socket:sendmsg(Child#child.sock, Msg) of @@ -789,7 +854,7 @@ drain_socket(State, #data{child = #child{sock = S, buf = Buf} = Child} = Data, A process_frames({restarting, _} = State, Data, Actions) -> {State, Data, Actions}; process_frames(State, #data{child = #child{buf = Buf} = Child} = Data, Actions) -> - case parse_frame(Buf) of + case py_child:parse_frame(Buf) of {ok, Frame, Rest} -> Data1 = Data#data{child = Child#child{buf = Rest}}, {State1, Data2, Actions1} = handle_frame(Frame, State, Data1, Actions), @@ -800,24 +865,6 @@ process_frames(State, #data{child = #child{buf = Buf} = Child} = Data, Actions) socket_broken({malformed_frame, Reason}, State, Data, Actions) end. -parse_frame(<>) -> - case Body of - <> -> - try - Term = case Payload of - <<>> -> undefined; - _ -> binary_to_term(Payload) - end, - {ok, {Id, Status, Term}, Rest} - catch - error:badarg -> {error, bad_etf} - end; - <<>> -> - {error, empty_body} - end; -parse_frame(_) -> - more. - handle_frame({Id, Status, Term}, State, #data{pending = Pending} = Data, Actions) when Status =:= ?STATUS_OK; Status =:= ?STATUS_ERROR -> @@ -904,7 +951,7 @@ run_callback({call, Name, Args}) -> T when is_tuple(T) -> tuple_to_list(T); _ -> [Args] end, - try py_callback:execute(to_bin(Name), ArgsList) of + try py_callback:execute(py_child:to_bin(Name), ArgsList) of {ok, Result} -> {?STATUS_OK, Result}; {error, {not_found, N}} -> @@ -979,9 +1026,11 @@ kill(Reason, State, #data{child = #child{port = Port, os_pid = OsPid}} = Data) - kill(_Reason, _State, _Data) -> keep_state_and_data. +kill_port(undefined, OsPid) -> + py_child:kill_os_pid(OsPid); kill_port(Port, OsPid) -> - case OsPid > 0 andalso erlang:port_info(Port) =/= undefined of - true -> _ = py_nif:os_kill(OsPid, 9), ok; + case erlang:port_info(Port) =/= undefined of + true -> py_child:kill_os_pid(OsPid); false -> ok end. @@ -1008,9 +1057,6 @@ enter_restarting(Reason, _State, Data, Actions) -> {{restarting, Reason}, Data#data{kill_target = undefined}, [{{timeout, kill}, cancel} | Actions]}. -exit_reason(Status) when Status > 128 -> {signal, Status - 128}; -exit_reason(Status) -> {exit_status, Status}. - %% The port reported the child's exit: fail what was in flight, then %% restart within the budget or stop. child_exited(Reason, State, #data{child = Child, opts = Opts} = Data0) -> @@ -1034,6 +1080,16 @@ child_exited(Reason, State, #data{child = Child, opts = Opts} = Data0) -> logger:warning("py_context ~p (isolated): child exited: ~p", [Data0#data.id, FailReason]) end, + case maps:get(session, Opts, false) of + true -> + %% No new child will come: kill/1 callers are answered now + [W ! {M, ok} || {W, M} <- Data1#data.kill_waiters], + {next_state, {exited, FailReason}, Data1#data{kill_waiters = []}}; + false -> + restart_or_stop(Reason, Data0, Data1) + end. + +restart_or_stop(Reason, Data0, #data{opts = Opts} = Data1) -> case maps:get(restart, Opts, true) andalso restart_allowed(Data1) of true -> Now = erlang:monotonic_time(millisecond), @@ -1083,30 +1139,50 @@ stop_child(#data{child = #child{port = Port, os_pid = OsPid} = Child} = Data, Ho case How of graceful -> _ = send_frame(Child, 0, ?STATUS_REQUEST, shutdown), - receive - {Port, {exit_status, _}} -> ok - after ?SHUTDOWN_GRACE_MS -> - kill_port(Port, OsPid), - wait_exit(Port) + case wait_exit(Child, ?SHUTDOWN_GRACE_MS) of + ok -> ok; + timeout -> kill_port(Port, OsPid), wait_exit(Child, 2000) end; + kill_nowait -> + kill_port(Port, OsPid); _ -> kill_port(Port, OsPid), - wait_exit(Port) + wait_exit(Child, 2000) end, close_child(Child), Data1 = fail_pending({child_exited, stopped}, Data), Data1#data{child = undefined}. -wait_exit(Port) -> +wait_exit(#child{port = undefined, os_pid = OsPid}, Timeout) -> + wait_forked_exit(OsPid, Timeout); +wait_exit(#child{port = Port}, Timeout) -> receive {Port, {exit_status, _}} -> ok - after 2000 -> - ok + after Timeout -> + timeout + end. + +%% The template reports a forked child's exit; if it is gone, the pid is +%% enough +wait_forked_exit(OsPid, Timeout) -> + receive + {py_session_exited, OsPid, _} -> ok + after min(Timeout, ?EXIT_PROBE_MS) -> + case py_child:os_pid_alive(OsPid) of + false -> ok; + true when Timeout =< ?EXIT_PROBE_MS -> timeout; + true -> wait_forked_exit(OsPid, Timeout - ?EXIT_PROBE_MS) + end end. close_child(#child{port = Port, sock = S, listener = L}) -> _ = socket:close(S), _ = socket:close(L), + close_port(Port). + +close_port(undefined) -> + ok; +close_port(Port) -> try port_close(Port) catch error:badarg -> ok end, ok. @@ -1134,16 +1210,8 @@ loop_exited(_Result, Data) -> %% Wire helpers %% --------------------------------------------------------------------------- -frame(Id, Status, Payload) -> - Body = <>, - <>. - send_frame(#child{sock = S}, Id, Status, Term) -> - case socket:send(S, frame(Id, Status, term_to_binary(Term))) of - ok -> ok; - {error, {Reason, _Rest}} -> {error, Reason}; - {error, Reason} -> {error, Reason} - end. + py_child:send_frame(S, Id, Status, Term). log_output(#data{id = Id, child = #child{os_pid = OsPid}}, Data) -> Lines = binary:split(Data, <<"\n">>, [global, trim_all]), @@ -1155,45 +1223,3 @@ log_event(#data{id = Id}, Level, Msg) -> error -> error; warning -> warning; debug -> debug; _ -> info end, logger:log(Lvl, "py_context ~p (isolated): ~s", [Id, Msg]). - -sock_dir() -> - Base = case os:getenv("TMPDIR") of - false -> "/tmp"; - T -> T - end, - Dir = filename:join(Base, "erlang_python_" ++ os:getpid()), - ok = filelib:ensure_dir(filename:join(Dir, "x")), - _ = file:change_mode(Dir, 8#700), - Dir. - -priv_dir() -> - case code:priv_dir(erlang_python) of - {error, bad_name} -> - filename:join(filename:dirname(filename:dirname(code:which(?MODULE))), "priv"); - Dir -> - Dir - end. - -rlimit_args(Opts) -> - Limits = maps:get(rlimits, Opts, #{}), - lists:append([case maps:get(K, Limits, undefined) of - undefined -> []; - V when is_integer(V), V >= 0 -> ["--rlimit-" ++ atom_to_list(K), integer_to_list(V)] - end || K <- [as, cpu, nofile]]). - -cgroup_args(Opts) -> - case maps:get(cgroup, Opts, undefined) of - undefined -> []; - Dir -> ["--cgroup", to_list(Dir)] - end. - -env_opt(Opts) -> - [{to_list(K), to_list(V)} || {K, V} <- maps:to_list(maps:get(env, Opts, #{}))]. - -to_bin(A) when is_atom(A) -> atom_to_binary(A, utf8); -to_bin(L) when is_list(L) -> unicode:characters_to_binary(L); -to_bin(B) when is_binary(B) -> B. - -to_list(A) when is_atom(A) -> atom_to_list(A); -to_list(B) when is_binary(B) -> unicode:characters_to_list(B); -to_list(L) when is_list(L) -> L. diff --git a/src/py_session.erl b/src/py_session.erl new file mode 100644 index 0000000..7b22a5f --- /dev/null +++ b/src/py_session.erl @@ -0,0 +1,182 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Isolated sessions: a fresh Python process per session, started from +%%% a prepared template. +%%% +%%% A template holds what every session starts with: interpreter, `sys.path', +%%% imports, preload code, environment, hash seed and limits. Each call to +%%% new/1 gives a new child process in which none of the state of another +%%% session exists; close/1 kills it. +%%% +%%% ``` +%%% {ok, T} = py_session:template(#{paths => [Dir], imports => [orders], +%%% hash_seed => 0}), +%%% {ok, S} = py_session:new(T), +%%% {ok, R} = py_context:call(S, orders, handle, [Event, State]), +%%% ok = py_session:close(S). +%%% ''' +%%% +%%% A session is an isolated context: every py_context function works on it. +%%% It is linked to the process that created it and is never restarted; if +%%% its child dies, requests answer `{error, Reason}' until it is closed. +%%% +%%% Template options: +%%%
      +%%%
    • `start' - `fork' (default): sessions are forked from a zygote that +%%% already ran the imports and preload. `spawn': each session starts +%%% a new interpreter; use it when the template cannot be forked +%%% (threads started at import, Objective-C on macOS). `reimport': +%%% no process per session; run/5 imports the function's module +%%% again in a fresh module dictionary on one of the template's +%%% `contexts' (`mode => worker | owngil'), as Temporal's workflow +%%% sandbox does. Modules in `imports' and `passthrough', and the +%%% standard library, are shared by every run.
      • +%%%
    • `python', `paths', `imports', `preload' - the prepared state.
      • +%%%
    • `env' - the environment of every session. Nothing is inherited +%%% from the VM unless `clear_env => false'.
      • +%%%
    • `hash_seed' - PYTHONHASHSEED shared by every session (default +%%% `random', chosen once per zygote).
      • +%%%
    • `zygotes' - zygotes forking in turn (default 1).
      • +%%%
    • `warm' - with `start => spawn', sessions kept started ahead.
      • +%%%
    • `rlimits', `cgroup', `start_timeout', `kill_after' - as for +%%% isolated contexts, applied to each session.
      • +%%%
    +-module(py_session). + +-export([template/1, + stop_template/1, + new/1, + new/2, + close/1, + run/4, + run/5, + refresh/1, + info/1]). + +-type template() :: pid(). +-type session() :: pid(). +-export_type([template/0, session/0]). + +%% @doc Build a template. With `start => fork' this starts the zygotes and +%% runs the imports and preload before returning. +-spec template(map()) -> {ok, template()} | {error, term()}. +template(Opts) when is_map(Opts) -> + py_session_sup:start_template(Opts). + +-spec stop_template(template()) -> ok. +stop_template(T) -> + py_session_sup:stop_template(T). + +%% @doc A new session: a fresh isolated process, linked to the caller. +-spec new(template()) -> {ok, session()} | {error, term()}. +new(T) -> + new(T, #{}). + +%% @doc A new session. `Opts' may carry `timeout' (milliseconds to get it). +-spec new(template(), map()) -> {ok, session()} | {error, term()}. +new(T, Opts) -> + Timeout = maps:get(timeout, Opts, 15000), + case py_session_template:checkout(T, Timeout) of + {ok, Ctx} -> + %% Started ahead by the template: take it over + MRef = erlang:monitor(process, Ctx), + Ctx ! {set_parent, self(), MRef, self()}, + receive + {MRef, ok} -> + erlang:demonitor(MRef, [flush]), + {ok, Ctx}; + {'DOWN', MRef, process, Ctx, _} -> + new(T, Opts) + after Timeout -> + erlang:demonitor(MRef, [flush]), + {error, timeout} + end; + {start, CtxOpts} -> + py_context:new(maps:merge(CtxOpts, maps:with([start_timeout], Opts))); + {reimport, _, _} -> + %% a re-import run is not a process: use run/5 + {error, {not_supported, reimport}}; + {error, _} = Err -> + Err + end. + +%% @doc Kill the session's process. A session is never reused. +-spec close(session()) -> ok. +close(S) -> + py_context:stop(S). + +%% @doc Run one call in a new session and close it. +-spec run(template(), atom() | binary(), atom() | binary(), list()) -> + {ok, term()} | {error, term()}. +run(T, Module, Func, Args) -> + run(T, Module, Func, Args, #{}). + +%% @doc Run one call in a new session and close it. `Opts': `kwargs', +%% `timeout' (for the call), plus the options of new/2. +-spec run(template(), atom() | binary(), atom() | binary(), list(), map()) -> + {ok, term()} | {error, term()}. +run(T, Module, Func, Args, Opts) -> + case py_session_template:checkout(T, maps:get(timeout, Opts, 15000)) of + {reimport, Ctx, Passthrough} -> + py_context:call(Ctx, '_erlang_impl._reimport', run, + [Passthrough, py_child:to_bin(Module), py_child:to_bin(Func), + Args, maps:get(kwargs, Opts, #{})], + #{}, maps:get(timeout, Opts, infinity)); + {ok, Ctx} -> + %% taken from the warm pool, handed over as new/2 does + run_in(T, Ctx, Module, Func, Args, Opts); + _ -> + run_new(T, Module, Func, Args, Opts) + end. + +run_in(T, Ctx, Module, Func, Args, Opts) -> + MRef = erlang:monitor(process, Ctx), + Ctx ! {set_parent, self(), MRef, self()}, + receive + {MRef, ok} -> + erlang:demonitor(MRef, [flush]), + try + py_context:call(Ctx, Module, Func, Args, maps:get(kwargs, Opts, #{}), + maps:get(timeout, Opts, infinity)) + after + close(Ctx) + end; + {'DOWN', MRef, process, Ctx, _} -> + run(T, Module, Func, Args, Opts) + end. + +run_new(T, Module, Func, Args, Opts) -> + case new(T, Opts) of + {ok, S} -> + try + py_context:call(S, Module, Func, Args, maps:get(kwargs, Opts, #{}), + maps:get(timeout, Opts, infinity)) + after + close(S) + end; + {error, _} = Err -> + Err + end. + +%% @doc Rebuild the template: new zygotes (code is imported again) or new +%% warm sessions. Live sessions are not touched. +-spec refresh(template()) -> ok | {error, term()}. +refresh(T) -> + py_session_template:refresh(T). + +%% @doc What the template runs: start mode, zygotes, sessions alive, forks. +-spec info(template()) -> map(). +info(T) -> + py_session_template:info(T). diff --git a/src/py_session_sup.erl b/src/py_session_sup.erl new file mode 100644 index 0000000..5baad8c --- /dev/null +++ b/src/py_session_sup.erl @@ -0,0 +1,51 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Supervisor of session templates. +%%% +%%% @private +-module(py_session_sup). + +-behaviour(supervisor). + +-export([start_link/0, start_template/1, stop_template/1]). +-export([init/1]). + +-spec start_link() -> {ok, pid()} | {error, term()}. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec start_template(map()) -> {ok, pid()} | {error, term()}. +start_template(Opts) -> + case supervisor:start_child(?MODULE, [Opts]) of + {ok, Pid} -> {ok, Pid}; + {error, _} = Err -> Err + end. + +-spec stop_template(pid()) -> ok. +stop_template(T) when is_pid(T) -> + _ = supervisor:terminate_child(?MODULE, T), + ok. + +init([]) -> + SupFlags = #{strategy => simple_one_for_one, intensity => 5, period => 10}, + ChildSpec = #{ + id => py_session_template, + start => {py_session_template, start_link, []}, + restart => temporary, + shutdown => 5000, + type => worker, + modules => [py_session_template] + }, + {ok, {SupFlags, [ChildSpec]}}. diff --git a/src/py_session_template.erl b/src/py_session_template.erl new file mode 100644 index 0000000..c30d0c5 --- /dev/null +++ b/src/py_session_template.erl @@ -0,0 +1,593 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc A prepared Python environment that sessions start from. +%%% +%%% `start => fork' keeps one or more zygotes (priv/py_zygote.py): Python +%%% processes that ran the template's imports and preload once and fork a +%%% fresh child per session. The zygote reports each child's exit here and +%%% the template passes it to the session's context as +%%% `{py_session_exited, OsPid, Code}'. +%%% +%%% `start => spawn' starts each session as a new isolated child and can keep +%%% `warm' of them started ahead of time; each is handed out once. +%%% +%%% `start => reimport' keeps `contexts' worker or owngil contexts; each +%%% py_session:run/5 runs in a fresh module dictionary on one of them +%%% (priv/_erlang_impl/_reimport.py), in the way of Temporal's workflow +%%% sandbox. There is no session process: new/1 is not available. +%%% +%%% A crashed zygote is rebuilt; the sessions it forked are separate +%%% processes and keep running. +%%% +%%% @private +%%% +%%% Owns: the zygote ports and control sockets, or the warm contexts. +%%% Talks to: `py_isolated' (fork requests, exit reports), `py_session'. +%%% Never: runs session requests; those go straight to the session context. +-module(py_session_template). + +-behaviour(gen_server). + +-export([start_link/1, + fork/4, + checkout/2, + refresh/1, + info/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). + +-define(STATUS_REQUEST, 0). +-define(STATUS_ERROR, 1). +-define(STATUS_OK, 2). +-define(STATUS_EVENT, 4). +-define(DEFAULT_START_TIMEOUT_MS, 10000). +-define(MAX_REBUILDS, 5). +-define(REBUILD_PERIOD_MS, 10000). + +-record(zygote, { + port :: port(), + os_pid :: non_neg_integer(), + listener :: socket:socket(), + sock :: socket:socket(), + buf = <<>> :: binary(), + info = #{} :: map() +}). + +-record(st, { + opts :: map(), + start :: fork | spawn | reimport, + %% start => reimport: the contexts runs go to, and what they share + contexts = [] :: [pid()], + passthrough = [] :: [binary()], + zygotes = [] :: [#zygote{}], + next = 0 :: non_neg_integer(), + next_id = 1 :: pos_integer(), + %% Id => From of fork requests the zygote has + pending = #{} :: #{pos_integer() => {gen_server:from(), pid()}}, + %% OsPid => context pid of live forked sessions + owners = #{} :: #{pos_integer() => pid()}, + %% start => spawn: contexts started ahead of time + warm = [] :: [pid()], + forks = 0 :: non_neg_integer(), + build_ms = 0 :: non_neg_integer(), + rebuilds = [] :: [integer()] +}). + +%% ============================================================================ +%% API +%% ============================================================================ + +-spec start_link(map()) -> {ok, pid()} | {error, term()}. +start_link(Opts) -> + gen_server:start_link(?MODULE, Opts, []). + +%% @doc Fork a session child that connects to SockPath. Called by the +%% session's context process, which receives the child's exit. +-spec fork(pid(), file:filename(), map(), timeout()) -> {ok, pos_integer()} | {error, term()}. +fork(T, SockPath, ForkOpts, Timeout) -> + try gen_server:call(T, {fork, SockPath, ForkOpts}, Timeout) + catch + exit:{timeout, _} -> {error, fork_timeout}; + exit:{noproc, _} -> {error, template_stopped}; + exit:{Reason, _} -> {error, {template_down, Reason}} + end. + +%% @doc A context started ahead of time (`{ok, Ctx}', linked to the +%% template until handed over), or the options to start one. +-spec checkout(pid(), timeout()) -> + {ok, pid()} | {start, map()} | {reimport, pid(), [binary()]} | {error, term()}. +checkout(T, Timeout) -> + try gen_server:call(T, checkout, Timeout) + catch + exit:{noproc, _} -> {error, template_stopped}; + exit:{Reason, _} -> {error, {template_down, Reason}} + end. + +-spec refresh(pid()) -> ok | {error, term()}. +refresh(T) -> + gen_server:call(T, refresh, infinity). + +-spec info(pid()) -> map(). +info(T) -> + gen_server:call(T, info). + +%% ============================================================================ +%% gen_server callbacks +%% ============================================================================ + +init(Opts) -> + process_flag(trap_exit, true), + case check_opts(Opts) of + ok -> + St = #st{opts = Opts, start = maps:get(start, Opts, fork)}, + case build(St) of + {ok, St1} -> {ok, St1}; + {error, _} = Err -> Err + end; + {error, _} = Err -> + Err + end. + +handle_call({fork, _Path, _ForkOpts}, _From, #st{start = spawn} = St) -> + {reply, {error, not_a_fork_template}, St}; +handle_call({fork, _Path, _ForkOpts}, _From, #st{zygotes = []} = St) -> + {reply, {error, no_zygote}, St}; +handle_call({fork, Path, ForkOpts}, {Pid, _} = From, St) -> + #st{zygotes = Zs, next = N, next_id = Id, pending = Pending} = St, + Z = lists:nth(N rem length(Zs) + 1, Zs), + Term = {fork, py_child:to_bin(Path), Pid, fork_opts(ForkOpts)}, + case py_child:send_frame(Z#zygote.sock, Id, ?STATUS_REQUEST, Term) of + ok -> + {noreply, St#st{next = N + 1, next_id = Id + 1, + pending = Pending#{Id => {From, Pid}}}}; + {error, Reason} -> + {reply, {error, {zygote_unreachable, Reason}}, St} + end; +handle_call(checkout, _From, #st{start = reimport, contexts = Cs, next = N, + passthrough = PT} = St) -> + {reply, {reimport, lists:nth(N rem length(Cs) + 1, Cs), PT}, St#st{next = N + 1}}; +handle_call(checkout, _From, #st{start = fork} = St) -> + {reply, {start, context_opts(St)}, St}; +handle_call(checkout, _From, #st{warm = [Ctx | Rest]} = St) -> + self() ! fill_warm, + {reply, {ok, Ctx}, St#st{warm = Rest}}; +handle_call(checkout, _From, #st{warm = []} = St) -> + self() ! fill_warm, + {reply, {start, context_opts(St)}, St}; +handle_call(refresh, _From, #st{start = fork, zygotes = Old} = St) -> + case build(St#st{zygotes = []}) of + {ok, St1} -> + %% The old zygotes exit when their control socket closes; the + %% sessions they forked keep running and are watched by pid + [retire(Z) || Z <- Old], + {reply, ok, St1#st{owners = #{}}}; + {error, Reason} -> + {reply, {error, Reason}, St} + end; +handle_call(refresh, _From, #st{start = reimport, contexts = Old} = St) -> + case build(St#st{contexts = []}) of + {ok, St1} -> + [begin unlink(C), py_context:stop(C) end || C <- Old], + {reply, ok, St1}; + {error, Reason} -> + {reply, {error, Reason}, St} + end; +handle_call(refresh, _From, #st{start = spawn, warm = Warm} = St) -> + [begin unlink(C), py_context:stop(C) end || C <- Warm], + self() ! fill_warm, + {reply, ok, St#st{warm = []}}; +handle_call(info, _From, St) -> + {reply, info_map(St), St}; +handle_call(_Other, _From, St) -> + {reply, {error, unknown_request}, St}. + +handle_cast(_Msg, St) -> + {noreply, St}. + +handle_info({'$socket', S, select, _}, St) -> + case lists:keyfind(S, #zygote.sock, St#st.zygotes) of + false -> {noreply, St}; + Z -> {noreply, drain(Z, St)} + end; +handle_info({'$socket', _, _, _}, St) -> + %% abort: the port's exit_status follows and drives the rebuild + {noreply, St}; +handle_info({Port, {data, Out}}, St) when is_port(Port) -> + [logger:info("py_session template ~p: ~s", [self(), L]) + || L <- binary:split(Out, <<"\n">>, [global, trim_all])], + {noreply, St}; +handle_info({Port, {exit_status, Status}}, St) when is_port(Port) -> + case lists:keytake(Port, #zygote.port, St#st.zygotes) of + {value, Z, Rest} -> zygote_exited(Z, py_child:exit_reason(Status), St#st{zygotes = Rest}); + false -> {noreply, St} + end; +handle_info(fill_warm, #st{start = spawn, warm = Warm, opts = Opts} = St) -> + case length(Warm) < maps:get(warm, Opts, 0) of + true -> + case py_context:start_link(erlang:unique_integer([positive]), isolated, + context_opts(St)) of + {ok, Ctx} -> + self() ! fill_warm, + {noreply, St#st{warm = Warm ++ [Ctx]}}; + {error, Reason} -> + logger:warning("py_session template ~p: warm session failed: ~p", + [self(), Reason]), + {noreply, St} + end; + false -> + {noreply, St} + end; +handle_info(fill_warm, St) -> + {noreply, St}; +handle_info({'EXIT', Pid, Reason}, #st{start = reimport, contexts = Cs} = St) -> + case lists:member(Pid, Cs) of + true -> + %% A context of the template stopped: put a new one in its place + logger:warning("py_session template ~p: context ~p exited: ~p", + [self(), Pid, Reason]), + case start_context(St#st.opts) of + {ok, C} -> + {noreply, St#st{contexts = [C | lists:delete(Pid, Cs)]}}; + {error, Why} -> + {stop, {context_restart_failed, Why}, St} + end; + false -> + {noreply, St} + end; +handle_info({'EXIT', Pid, _Reason}, #st{warm = Warm} = St) -> + %% A warm session died before anyone took it + {noreply, St#st{warm = lists:delete(Pid, Warm)}}; +handle_info(_Msg, St) -> + {noreply, St}. + +terminate(_Reason, #st{zygotes = Zs, warm = Warm, contexts = Cs}) -> + [retire(Z) || Z <- Zs], + [py_context:stop(C) || C <- Warm ++ Cs], + ok. + +%% ============================================================================ +%% Building +%% ============================================================================ + +check_opts(Opts) -> + Checks = [ + fun() -> case maps:get(start, Opts, fork) of + S when S =:= fork; S =:= spawn; S =:= reimport -> ok; + S -> {error, {badarg, {start, S}}} + end end, + fun() -> case {maps:get(start, Opts, fork), maps:get(mode, Opts, worker)} of + {reimport, M} when M =:= worker; M =:= owngil -> ok; + {reimport, M} -> {error, {badarg, {mode, M}}}; + _ -> ok + end end, + fun() -> case maps:get(contexts, Opts, 1) of + N when is_integer(N), N >= 1 -> ok; + N -> {error, {badarg, {contexts, N}}} + end end, + fun() -> case maps:get(zygotes, Opts, 1) of + N when is_integer(N), N >= 1 -> ok; + N -> {error, {badarg, {zygotes, N}}} + end end, + fun() -> case maps:get(warm, Opts, 0) of + N when is_integer(N), N >= 0 -> ok; + N -> {error, {badarg, {warm, N}}} + end end, + fun() -> py_child:check_env_opts(env_opts(Opts)) end, + fun() -> case {maps:get(cgroup, Opts, undefined), os:type()} of + {undefined, _} -> ok; + {_, {unix, linux}} -> ok; + {_, {unix, Os}} -> {error, {cgroup_unsupported, Os}} + end end, + fun() -> case py_child:python_executable(Opts) of + {error, _} = Err -> Err; + _ -> ok + end end + ], + lists:foldl(fun(Check, ok) -> Check(); (_, Err) -> Err end, ok, Checks). + +%% Sessions see only the template's env unless it asks otherwise +env_opts(Opts) -> + maps:merge(#{clear_env => true}, maps:with([env, clear_env, hash_seed], Opts)). + +build(#st{start = spawn} = St) -> + self() ! fill_warm, + {ok, St}; +build(#st{start = reimport, opts = Opts} = St) -> + T0 = erlang:monotonic_time(millisecond), + Started = [start_context(Opts) || _ <- lists:seq(1, maps:get(contexts, Opts, 1))], + case [E || {error, _} = E <- Started] of + [] -> + PT = lists:usort([py_child:to_bin(M) || M <- maps:get(imports, Opts, []) + ++ maps:get(passthrough, Opts, [])]), + {ok, St#st{contexts = [C || {ok, C} <- Started], passthrough = PT, + build_ms = erlang:monotonic_time(millisecond) - T0}}; + [Err | _] -> + [py_context:stop(C) || {ok, C} <- Started], + Err + end; +build(#st{opts = Opts} = St) -> + T0 = erlang:monotonic_time(millisecond), + case build_zygotes(maps:get(zygotes, Opts, 1), St#st.opts, []) of + {ok, Zs} -> + {ok, St#st{zygotes = Zs, build_ms = erlang:monotonic_time(millisecond) - T0}}; + {error, _} = Err -> + Err + end. + +build_zygotes(0, _Opts, Acc) -> + {ok, lists:reverse(Acc)}; +build_zygotes(N, Opts, Acc) -> + case start_zygote(Opts) of + {ok, Z} -> + build_zygotes(N - 1, Opts, [Z | Acc]); + {error, _} = Err -> + [retire(Z) || Z <- Acc], + Err + end. + +start_zygote(Opts) -> + Python = py_child:python_executable(Opts), + Path = py_child:new_sock_path("tpl_"), + Timeout = maps:get(start_timeout, Opts, ?DEFAULT_START_TIMEOUT_MS), + case py_child:listen(Path) of + {ok, L} -> + Script = filename:join(py_child:priv_dir(), "py_zygote.py"), + Port = open_port({spawn_executable, Python}, + [exit_status, stderr_to_stdout, binary, use_stdio, + {args, [Script, Path]}, + {env, py_child:port_env(env_opts(Opts))}]), + OsPid = case erlang:port_info(Port, os_pid) of + {os_pid, P} -> P; + _ -> 0 + end, + Result = case py_child:accept(L, {port, Port}, Timeout) of + {ok, S} -> + py_child:tune_socket(S), + Z = #zygote{port = Port, os_pid = OsPid, listener = L, sock = S}, + prepare(Z, Opts, Timeout); + {error, _} = Err -> + Err + end, + py_child:delete_file(Path), + case Result of + {ok, _} = Ok -> + Ok; + {error, Reason} -> + socket:close(L), + py_child:kill_os_pid(OsPid), + close_port(Port), + {error, {template_failed, Reason}} + end; + {error, Reason} -> + {error, {template_failed, Reason}} + end. + +%% A context for re-import runs: the paths, the imports (shared by every +%% run) and the preload are applied once. +start_context(Opts) -> + CtxOpts = maps:merge(#{mode => maps:get(mode, Opts, worker)}, + maps:with([preload], Opts)), + case py_context:new(CtxOpts) of + {ok, C} -> + Paths = [py_child:to_list(P) || P <- maps:get(paths, Opts, [])], + Setup = iolist_to_binary(io_lib:format( + "import sys, importlib +" + "for _p in reversed(~p): +" + " if _p not in sys.path: sys.path.insert(0, _p) +" + "for _m in ~p: importlib.import_module(_m) +" + "import _erlang_impl._reimport +", + [Paths, [py_child:to_list(M) || M <- maps:get(imports, Opts, [])]])), + case py_context:exec(C, Setup) of + ok -> + {ok, C}; + {error, Reason} -> + py_context:stop(C), + {error, {template_failed, {init_failed, Reason}}} + end; + {error, Reason} -> + {error, {template_failed, Reason}} + end. + +%% Wait for `ready', run the imports and preload, then arm the socket. +prepare(Z, Opts, Timeout) -> + case recv_sync(Z, Timeout) of + {ok, {0, ?STATUS_EVENT, {ready, Info}}, Z1} -> + Paths = [py_child:to_bin(P) || P <- py_import:all_paths()] + ++ [py_child:to_bin(P) || P <- maps:get(paths, Opts, [])], + Imports = lists:usort([py_child:to_bin(M) || {M, _} <- py_import:all_imports()] + ++ [py_child:to_bin(M) || M <- maps:get(imports, Opts, [])]), + Preload = iolist_to_binary([preload_code(), <<"\n">>, + maps:get(preload, Opts, <<>>)]), + ok = py_child:send_frame(Z1#zygote.sock, 1, ?STATUS_REQUEST, + {init, Paths, Imports, Preload}), + case recv_sync(Z1, Timeout) of + {ok, {1, ?STATUS_OK, _}, Z2} -> + {ok, arm(Z2#zygote{info = Info})}; + {ok, {1, ?STATUS_ERROR, Why}, _} -> + {error, {init_failed, Why}}; + {ok, Other, _} -> + {error, {unexpected_frame, Other}}; + {error, _} = Err -> + Err + end; + {ok, Other, _} -> + {error, {unexpected_frame, Other}}; + {error, _} = Err -> + Err + end. + +preload_code() -> + try py_preload:get_code() of + Code when is_binary(Code) -> Code; + _ -> <<>> + catch + _:_ -> <<>> + end. + +recv_sync(#zygote{sock = S, buf = Buf, port = Port} = Z, Timeout) -> + case py_child:parse_frame(Buf) of + {ok, Frame, Rest} -> + {ok, Frame, Z#zygote{buf = Rest}}; + more -> + case socket:recv(S, 0, Timeout) of + {ok, Data} -> + recv_sync(Z#zygote{buf = <>}, Timeout); + {error, Reason} -> + {error, {Reason, drain_output(Port, [])}} + end; + {error, _} = Err -> + Err + end. + +drain_output(Port, Acc) -> + receive + {Port, {data, D}} -> drain_output(Port, [D | Acc]) + after 50 -> + iolist_to_binary(lists:reverse(Acc)) + end. + +%% Arm the select; frames already buffered are processed first +arm(#zygote{} = Z) -> + self() ! {'$socket', Z#zygote.sock, select, arm}, + Z. + +retire(#zygote{sock = S, listener = L, port = Port}) -> + _ = socket:close(S), + _ = socket:close(L), + close_port(Port). + +close_port(Port) -> + try port_close(Port) catch error:badarg -> ok end, + ok. + +%% ============================================================================ +%% Frames from a zygote +%% ============================================================================ + +drain(#zygote{sock = S, buf = Buf} = Z, St) -> + case socket:recv(S, 0, nowait) of + {ok, Data} -> + drain(Z#zygote{buf = <>}, St); + {select, _} -> + frames(Z, St); + {error, _} -> + %% The port's exit_status follows + frames(Z, St) + end. + +frames(#zygote{buf = Buf} = Z, St) -> + case py_child:parse_frame(Buf) of + {ok, Frame, Rest} -> + St1 = frame(Frame, St), + frames(Z#zygote{buf = Rest}, St1); + more -> + store(Z, St); + {error, Reason} -> + logger:error("py_session template ~p: bad frame from zygote ~p: ~p", + [self(), Z#zygote.os_pid, Reason]), + py_child:kill_os_pid(Z#zygote.os_pid), + store(Z#zygote{buf = <<>>}, St) + end. + +store(Z, #st{zygotes = Zs} = St) -> + St#st{zygotes = lists:keyreplace(Z#zygote.sock, #zygote.sock, Zs, Z)}. + +frame({Id, Status, Term}, #st{pending = Pending, owners = Owners} = St) + when Status =:= ?STATUS_OK; Status =:= ?STATUS_ERROR -> + case maps:take(Id, Pending) of + {{From, Pid}, Rest} when Status =:= ?STATUS_OK -> + gen_server:reply(From, {ok, Term}), + St#st{pending = Rest, owners = Owners#{Term => Pid}, forks = St#st.forks + 1}; + {{From, _Pid}, Rest} -> + gen_server:reply(From, {error, {fork_failed, Term}}), + St#st{pending = Rest}; + error -> + St + end; +frame({_, ?STATUS_EVENT, {exited, OsPid, Code}}, #st{owners = Owners} = St) -> + case maps:take(OsPid, Owners) of + {Pid, Rest} -> + Pid ! {py_session_exited, OsPid, Code}, + St#st{owners = Rest}; + error -> + St + end; +frame(_, St) -> + St. + +%% ============================================================================ +%% Zygote failure +%% ============================================================================ + +zygote_exited(Z, Reason, #st{pending = Pending} = St) -> + logger:warning("py_session template ~p: zygote ~p exited: ~p", + [self(), Z#zygote.os_pid, Reason]), + retire(Z), + %% Forks it had not answered fail; the sessions it forked keep running + %% and their contexts watch them by pid + [gen_server:reply(From, {error, {zygote_exited, Reason}}) || {From, _} <- maps:values(Pending)], + St1 = St#st{pending = #{}}, + Now = erlang:monotonic_time(millisecond), + Recent = [T || T <- St1#st.rebuilds, Now - T =< ?REBUILD_PERIOD_MS], + case length(Recent) < ?MAX_REBUILDS of + true -> + case start_zygote(St1#st.opts) of + {ok, NewZ} -> + {noreply, St1#st{zygotes = St1#st.zygotes ++ [NewZ], + rebuilds = [Now | Recent]}}; + {error, Why} -> + {stop, {zygote_rebuild_failed, Why}, St1} + end; + false -> + {stop, {zygote_exited, Reason}, St1} + end. + +%% ============================================================================ +%% Helpers +%% ============================================================================ + +%% Options of the isolated context behind each session +context_opts(#st{start = fork, opts = Opts}) -> + maps:merge(maps:with([rlimits, cgroup, start_timeout, kill_after], Opts), + #{mode => isolated, session => true, restart => false, + origin => {fork, self()}}); +context_opts(#st{start = spawn, opts = Opts}) -> + maps:merge(maps:with([python, paths, preload, rlimits, cgroup, start_timeout, + kill_after], Opts), + (env_opts(Opts))#{mode => isolated, session => true, restart => false, + origin => spawn}). + +fork_opts(ForkOpts) -> + maps:fold(fun(rlimits, V, Acc) -> Acc#{rlimits => V}; + (cgroup, V, Acc) -> Acc#{cgroup => py_child:to_bin(V)}; + (cd, V, Acc) -> Acc#{cd => py_child:to_bin(V)}; + (_, _, Acc) -> Acc + end, #{}, ForkOpts). + +info_map(#st{start = Start, zygotes = Zs, owners = Owners, forks = Forks, + build_ms = BuildMs, warm = Warm, opts = Opts, contexts = Cs}) -> + #{start => Start, + contexts => Cs, + zygotes => [Info#{os_pid => P} || #zygote{os_pid = P, info = Info} <- Zs], + sessions => map_size(Owners), + forks => Forks, + build_ms => BuildMs, + warm => length(Warm), + hash_seed => maps:get(hash_seed, Opts, random)}. diff --git a/test/coverage_audit.md b/test/coverage_audit.md index bb8a8ca..25076ee 100644 --- a/test/coverage_audit.md +++ b/test/coverage_audit.md @@ -19,6 +19,10 @@ suite is visible. `scripts/check_code_map.sh` requires a row per module. | `py_venv` | `py_venv_SUITE` | | `py_shared_dict` | `py_SUITE` | | `py_isolated` | `py_isolated_*_SUITE` | +| `py_child` | `py_isolated_*_SUITE`, `py_session_SUITE` | +| `py_session` | `py_session_SUITE`, `py_session_stress_SUITE`, `py_session_soak_SUITE` | +| `py_session_template` | `py_session_SUITE`, `py_session_stress_SUITE`, `py_session_soak_SUITE` | +| `py_session_sup` | (through `py_session_SUITE`) | | `py_context_router` | `py_context_router_SUITE`, `py_pool_SUITE` | | `py_context_sup` | (through the above) | | `py_context_init` | (through the above) | diff --git a/test/py_isolated_SUITE.erl b/test/py_isolated_SUITE.erl index 937bd9d..0b6ba29 100644 --- a/test/py_isolated_SUITE.erl +++ b/test/py_isolated_SUITE.erl @@ -64,6 +64,9 @@ test_startup_error_reported/1, test_cgroup_option_platform/1, test_env_option/1, + test_clear_env_option/1, + test_hash_seed_option/1, + test_bad_env_options/1, test_preload_option/1 ]). @@ -119,6 +122,9 @@ groups() -> test_startup_error_reported, test_cgroup_option_platform, test_env_option, + test_clear_env_option, + test_hash_seed_option, + test_bad_env_options, test_preload_option ], [{worker, [], RoundTrip}, @@ -786,6 +792,51 @@ test_env_option(Config) -> {ok, <<"yes">>} = py_context:eval(C, <<"__import__('os').environ.get('PY_ISOLATED_PROBE')">>), stop(C). +%% The VM's environment reaches the child unless clear_env is set; with it +%% the child sees only what `env' names. +test_clear_env_option(Config) -> + true = os:putenv("PY_ISOLATED_LEAK", "from-the-vm"), + try + Probe = <<"(os.environ.get('PY_ISOLATED_LEAK'), os.environ.get('PY_ISOLATED_KEEP'))">>, + Inherit = new_ctx(Config, #{env => #{"PY_ISOLATED_KEEP" => "1"}}), + ok = py_context:exec(Inherit, <<"import os">>), + {ok, {<<"from-the-vm">>, <<"1">>}} = py_context:eval(Inherit, Probe), + stop(Inherit), + Clean = new_ctx(Config, #{clear_env => true, env => #{"PY_ISOLATED_KEEP" => "1"}}), + ok = py_context:exec(Clean, <<"import os">>), + {ok, {none, <<"1">>}} = py_context:eval(Clean, Probe), + {ok, Names} = py_context:eval(Clean, <<"sorted(os.environ)">>), + %% Python itself may add a few (e.g. __CF_USER_TEXT_ENCODING on macOS) + false = lists:member(<<"HOME">>, Names), + false = lists:member(<<"PATH">>, Names), + stop(Clean) + after + os:unsetenv("PY_ISOLATED_LEAK") + end. + +%% Same seed, same str hashes and set order in two separate children; another +%% seed changes them. +test_hash_seed_option(Config) -> + Probe = <<"(hash('erlang-python'), repr(list({'a','b','c','d','e','f','g','h'})))">>, + Run = fun(Seed) -> + C = new_ctx(Config, #{hash_seed => Seed}), + {ok, R} = py_context:eval(C, Probe), + {ok, Env} = py_context:eval(C, <<"__import__('os').environ.get('PYTHONHASHSEED')">>), + stop(C), + {R, Env} + end, + {A, <<"7">>} = Run(7), + {A, <<"7">>} = Run(7), + {B, <<"8">>} = Run(8), + true = A =/= B, + ok. + +test_bad_env_options(_Config) -> + {error, {badarg, {hash_seed, -1}}} = py_context:new(#{mode => isolated, hash_seed => -1}), + {error, {badarg, {hash_seed, 1 bsl 32}}} = py_context:new(#{mode => isolated, hash_seed => 1 bsl 32}), + {error, {badarg, {clear_env, yes}}} = py_context:new(#{mode => isolated, clear_env => yes}), + ok. + test_preload_option(Config) -> C = new_ctx(Config, #{preload => <<"preloaded = 'yes'">>}), {ok, <<"yes">>} = py_context:eval(C, <<"preloaded">>), diff --git a/test/py_reentrant_SUITE.erl b/test/py_reentrant_SUITE.erl index 47af7be..599faf1 100644 --- a/test/py_reentrant_SUITE.erl +++ b/test/py_reentrant_SUITE.erl @@ -31,7 +31,9 @@ test_call_reentrant_depths/1, test_call_reentrant_repeated/1, test_call_sequential_callbacks/1, - test_readme_reentrant_example/1 + test_readme_reentrant_example/1, + test_owngil_call_reentrant_depths/1, + test_owngil_concurrent_reentrant/1 ]). all() -> @@ -52,7 +54,9 @@ all() -> test_call_reentrant_depths, test_call_reentrant_repeated, test_call_sequential_callbacks, - test_readme_reentrant_example + test_readme_reentrant_example, + test_owngil_call_reentrant_depths, + test_owngil_concurrent_reentrant ]. init_per_suite(Config) -> @@ -83,6 +87,7 @@ end_per_testcase(_TestCase, _Config) -> try py:unregister_function(etf_probe_novel) catch _:_ -> ok end, try py:unregister_function(rs_double) catch _:_ -> ok end, try py:unregister_function(nest_step) catch _:_ -> ok end, + try py:unregister_function(owngil_nest) catch _:_ -> ok end, ok. %%% ============================================================================ @@ -115,6 +120,65 @@ test_call_reentrant_depths(_Config) -> py:unregister_function(nest_step), ok. +%%% owngil: the context thread used to block on the callback pipe, so a +%%% callback calling back into the same owngil context queued behind the +%%% request waiting for it and the chain hung until the timeout. + +-define(OWNGIL_DOWN, <<" +import erlang +def odown(ctx, n, d): + if d == 0: + return n + return erlang.call('owngil_nest', ctx, n, d) +">>). + +owngil_ctx() -> + {ok, C} = py_context:new(#{mode => owngil}), + ok = py_context:exec(C, ?OWNGIL_DOWN), + C. + +register_owngil_nest() -> + py:register_function(owngil_nest, fun([Ctx, N, D]) -> + {ok, R} = py_context:call(Ctx, '__main__', odown, [Ctx, N + 1, D - 1], #{}, 10000), + R + end). + +%% @doc py_context:call on an owngil context -> erlang.call -> the same +%% context, 1 to 5 deep, then repeated. +test_owngil_call_reentrant_depths(_Config) -> + case py_nif:owngil_supported() of + false -> + {skip, "OWN_GIL requires Python 3.14+"}; + true -> + register_owngil_nest(), + C = owngil_ctx(), + [{ok, D} = py_context:call(C, '__main__', odown, [C, 0, D], #{}, 10000) + || D <- [1, 2, 3, 5]], + [{ok, 2} = py_context:call(C, '__main__', odown, [C, 0, 2], #{}, 10000) + || _ <- lists:seq(1, 20)], + py_context:stop(C) + end. + +%% @doc Several owngil contexts nesting at once, each into itself. +test_owngil_concurrent_reentrant(_Config) -> + case py_nif:owngil_supported() of + false -> + {skip, "OWN_GIL requires Python 3.14+"}; + true -> + register_owngil_nest(), + Ctxs = [owngil_ctx() || _ <- lists:seq(1, 4)], + Self = self(), + Pids = [spawn_link(fun() -> + C = lists:nth(I rem 4 + 1, Ctxs), + Self ! {done, [py_context:call(C, '__main__', odown, [C, 0, 3], #{}, 10000) + || _ <- lists:seq(1, 25)]} + end) || I <- lists:seq(1, 8)], + [receive {done, R} -> R = lists:duplicate(25, {ok, 3}) after 60000 -> ct:fail(timeout) end + || _ <- Pids], + [py_context:stop(C) || C <- Ctxs], + ok + end. + %% @doc One depth, many times: the nested py:call must not depend on which %% context the scheduler picks. test_call_reentrant_repeated(_Config) -> diff --git a/test/py_session_SUITE.erl b/test/py_session_SUITE.erl new file mode 100644 index 0000000..458f37f --- /dev/null +++ b/test/py_session_SUITE.erl @@ -0,0 +1,702 @@ +%%% @doc Isolated sessions (py_session): a fresh process per session, started +%%% from a template, by fork from a zygote or by spawning a child. +%%% +%%% The `fork' and `spawn' groups run the same cases; `fork_only' and +%%% `spawn_only' cover what differs. +-module(py_session_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, groups/0, init_per_suite/1, end_per_suite/1, + init_per_group/2, end_per_group/2, end_per_testcase/2]). + +%% logger handler used by test_print_is_logged +-export([log/2]). + +-export([ + test_sessions_are_isolated/1, + test_same_start_state/1, + test_env_replaced/1, + test_hash_seed/1, + test_random_differs/1, + test_context_api/1, + test_reentrance_depth/1, + test_concurrent_reentrance/1, + test_cross_session_call/1, + test_interrupt_nested_call/1, + test_session_crash/1, + test_kill_session/1, + test_close_reaps_and_cleans/1, + test_caller_crash_stops_session/1, + test_run_helper/1, + test_print_is_logged/1, + test_rlimits_apply/1, + test_refresh_picks_new_code/1, + test_bad_template_options/1, + test_zygote_crash_rebuilds/1, + test_thread_at_import_refused/1, + test_erlang_call_during_preload/1, + test_info_counts_forks/1, + test_late_exit_report/1, + test_warm_pool/1, + test_thread_at_import_spawned/1, + test_reimport_runs_are_isolated/1, + test_reimport_shares_stdlib_and_imports/1, + test_reimport_concurrent_runs/1, + test_reimport_reentrance/1, + test_reimport_typing_and_pickle/1, + test_reimport_errors/1, + test_reimport_no_session_process/1, + test_reimport_refresh/1, + test_reimport_bad_options/1 +]). + +-define(MOD, py_test_session). + +all() -> + [{group, fork}, {group, spawn}, {group, fork_only}, {group, spawn_only}, + {group, reimport_worker}, {group, reimport_owngil}]. + +groups() -> + Common = [ + test_sessions_are_isolated, + test_same_start_state, + test_env_replaced, + test_hash_seed, + test_random_differs, + test_context_api, + test_reentrance_depth, + test_concurrent_reentrance, + test_cross_session_call, + test_interrupt_nested_call, + test_session_crash, + test_kill_session, + test_close_reaps_and_cleans, + test_caller_crash_stops_session, + test_run_helper, + test_print_is_logged, + test_rlimits_apply, + test_refresh_picks_new_code, + test_bad_template_options + ], + Reimport = [ + test_reimport_runs_are_isolated, + test_reimport_shares_stdlib_and_imports, + test_reimport_concurrent_runs, + test_reimport_reentrance, + test_reimport_typing_and_pickle, + test_reimport_errors, + test_reimport_no_session_process, + test_reimport_refresh, + test_reimport_bad_options + ], + [{fork, [], Common}, + {spawn, [], Common}, + {fork_only, [], [test_zygote_crash_rebuilds, + test_thread_at_import_refused, + test_erlang_call_during_preload, + test_info_counts_forks, + test_late_exit_report]}, + {spawn_only, [], [test_warm_pool, + test_thread_at_import_spawned]}, + {reimport_worker, [], Reimport}, + {reimport_owngil, [], Reimport}]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + TestDir = filename:join(code:lib_dir(erlang_python), "test"), + py:register_function(sess_reenter, fun([Ctx, N]) -> + {ok, R} = py_context:call(Ctx, ?MOD, reenter, [N - 1]), + R + end), + py:register_function(sess_nested_sleep, fun([Ctx, Seconds]) -> + py_context:call(Ctx, ?MOD, sleep, [Seconds], #{}, 200) + end), + [{test_dir, TestDir} | Config]. + +end_per_suite(_Config) -> + py:unregister_function(sess_reenter), + py:unregister_function(sess_nested_sleep), + ok. + +init_per_group(G, Config) when G =:= fork; G =:= fork_only -> + [{start, fork} | Config]; +init_per_group(reimport_worker, Config) -> + [{start, reimport}, {mode, worker} | Config]; +init_per_group(reimport_owngil, Config) -> + case py_nif:owngil_supported() of + true -> [{start, reimport}, {mode, owngil} | Config]; + false -> {skip, "OWN_GIL requires Python 3.14+"} + end; +init_per_group(_G, Config) -> + [{start, spawn} | Config]. + +end_per_group(_G, _Config) -> + ok. + +end_per_testcase(_Case, _Config) -> + [py_session:stop_template(T) || {_, T, _, _} <- supervisor:which_children(py_session_sup)], + ok. + +%%% ============================================================================ +%%% Common cases +%%% ============================================================================ + +%% Nothing a session changes is seen by the next one: module globals, +%% sys.modules, os.environ, threads, files in its directory. +test_sessions_are_isolated(Config) -> + T = template(Config), + Prev = lists:foldl(fun(I, Prev) -> + {ok, S} = py_session:new(T), + Mark = integer_to_binary(I), + {ok, Prev} = py_context:call(S, ?MOD, peek, []), + {ok, Mark} = py_context:call(S, ?MOD, mark, [Mark]), + py_session:close(S), + none + end, none, lists:seq(1, 100)), + none = Prev, + {ok, A} = py_session:new(T), + {ok, true} = py_context:call(A, ?MOD, put_module, [<<"sess_marker">>]), + {ok, <<"1">>} = py_context:call(A, ?MOD, env_set, [<<"SESS_WRITE">>, <<"1">>]), + {ok, N0} = py_context:call(A, ?MOD, thread_count, []), + {ok, _} = py_context:call(A, ?MOD, start_thread, []), + {ok, [<<"f">>]} = py_context:call(A, ?MOD, write_file, [<<"f">>, <<"x">>]), + {ok, B} = py_session:new(T), + {ok, false} = py_context:call(B, ?MOD, has_module, [<<"sess_marker">>]), + {ok, none} = py_context:call(B, ?MOD, env_get, [<<"SESS_WRITE">>]), + {ok, N0} = py_context:call(B, ?MOD, thread_count, []), + {ok, []} = py_context:call(B, ?MOD, list_cwd, []), + {ok, PA} = py_context:call(A, ?MOD, pid, []), + {ok, PB} = py_context:call(B, ?MOD, pid, []), + true = PA =/= PB, + py_session:close(A), + py_session:close(B). + +%% Every session starts with the template's imports and preload; with fork +%% they ran once, in the zygote. +test_same_start_state(Config) -> + T = template(Config, #{preload => <<"PRELOADED = 'yes'">>}), + {ok, A} = py_session:new(T), + {ok, B} = py_session:new(T), + {ok, <<"yes">>} = py_context:eval(A, <<"PRELOADED">>), + {ok, <<"yes">>} = py_context:eval(B, <<"PRELOADED">>), + {ok, ImportedA} = py_context:call(A, ?MOD, imported_in, []), + {ok, ImportedB} = py_context:call(B, ?MOD, imported_in, []), + {ok, PidA} = py_context:call(A, ?MOD, pid, []), + case ?config(start, Config) of + fork -> + %% imported once, by the zygote, before either session existed + ImportedA = ImportedB, + true = ImportedA =/= PidA; + spawn -> + ImportedA = PidA + end, + py_session:close(A), + py_session:close(B). + +%% The VM's environment does not reach a session: it sees the template's env. +test_env_replaced(Config) -> + true = os:putenv("SESS_VM_ONLY", "leak"), + try + T = template(Config, #{env => #{"SESS_DECLARED" => "1"}}), + {ok, S} = py_session:new(T), + {ok, none} = py_context:call(S, ?MOD, env_get, [<<"SESS_VM_ONLY">>]), + {ok, <<"1">>} = py_context:call(S, ?MOD, env_get, [<<"SESS_DECLARED">>]), + {ok, Names} = py_context:call(S, ?MOD, env_names, []), + false = lists:member(<<"HOME">>, Names), + py_session:close(S) + after + os:unsetenv("SESS_VM_ONLY") + end. + +%% One hash seed per template: every session hashes strings and orders sets +%% the same way; another seed changes it. +test_hash_seed(Config) -> + T1 = template(Config, #{hash_seed => 11}), + T2 = template(Config, #{hash_seed => 12}), + Probe = fun(T) -> + {ok, S} = py_session:new(T), + {ok, R} = py_context:call(S, ?MOD, hash_probe, []), + py_session:close(S), + R + end, + P = Probe(T1), + P = Probe(T1), + true = P =/= Probe(T2), + ok. + +%% A forked session does not inherit the zygote's random state as is. +test_random_differs(Config) -> + T = template(Config), + Values = [begin + {ok, S} = py_session:new(T), + {ok, V} = py_context:call(S, ?MOD, random_value, []), + py_session:close(S), + V + end || _ <- lists:seq(1, 5)], + 5 = length(lists:usort(Values)), + ok. + +%% A session is an isolated context: the whole context API works on it. +test_context_api(Config) -> + T = template(Config), + {ok, S} = py_session:new(T), + {ok, 4} = py_context:eval(S, <<"2 + 2">>), + ok = py_context:exec(S, <<"x = 21">>), + {ok, 42} = py_context:eval(S, <<"x * 2">>), + {ok, 3} = py_context:call(S, ?MOD, async_add, [1, 2]), + %% timeout interrupts the call; the session keeps working + {error, timeout} = py_context:call(S, ?MOD, sleep, [30], #{}, 300), + {ok, 1} = py_context:eval(S, <<"1">>), + {ok, LSock} = gen_tcp:listen(0, [{ip, {127, 0, 0, 1}}]), + {ok, Fd} = inet:getfd(LSock), + {ok, ChildFd} = py_context:pass_fd(S, Fd), + {ok, true} = py_context:call(S, ?MOD, fd_is_open, [ChildFd]), + gen_tcp:close(LSock), + ok = py_context:start_loop(S), + {ok, 7} = py_context:submit_await(S, ?MOD, async_add, [3, 4]), + ok = py_context:stop_loop(S), + {ok, Info} = py_context:child_info(S), + true = is_integer(maps:get(os_pid, Info)), + py_session:close(S). + +%% Python -> Erlang -> the same session -> Erlang -> ... ten levels deep. +test_reentrance_depth(Config) -> + T = template(Config), + {ok, S} = py_session:new(T), + {ok, 10} = py_context:call(S, ?MOD, reenter, [10]), + {ok, 1} = py_context:eval(S, <<"1">>), + py_session:close(S). + +test_concurrent_reentrance(Config) -> + T = template(Config), + Self = self(), + Pids = [spawn_link(fun() -> + {ok, S} = py_session:new(T), + R = [py_context:call(S, ?MOD, reenter, [5]) || _ <- lists:seq(1, 10)], + py_session:close(S), + Self ! {self(), R} + end) || _ <- lists:seq(1, 16)], + [receive {P, R} -> R = lists:duplicate(10, {ok, 5}) after 30000 -> ct:fail(timeout) end + || P <- Pids], + ok. + +%% A callback of session A calls into session B. +test_cross_session_call(Config) -> + T = template(Config), + {ok, A} = py_session:new(T), + {ok, B} = py_session:new(T), + {ok, <<"b">>} = py_context:call(B, ?MOD, mark, [<<"b">>]), + py:register_function(sess_cross, fun([_N]) -> + {ok, V} = py_context:call(B, ?MOD, peek, []), + V + end), + try + {ok, <<"b">>} = py_context:call(A, ?MOD, cross, [1]), + {ok, none} = py_context:call(A, ?MOD, peek, []) + after + py:unregister_function(sess_cross) + end, + py_session:close(A), + py_session:close(B). + +%% A nested call that times out is interrupted alone; the outer call it +%% runs inside of completes. +test_interrupt_nested_call(Config) -> + T = template(Config), + {ok, S} = py_session:new(T), + %% {error, timeout} crossed into Python and back: atoms are strings there + {ok, {<<"outer-done">>, {<<"error">>, <<"timeout">>}}} = + py_context:call(S, ?MOD, nested_sleep_then, [30]), + {ok, 1} = py_context:eval(S, <<"1">>), + py_session:close(S). + +%% A session whose child dies answers with the reason until closed; the +%% template and other sessions are untouched. +test_session_crash(Config) -> + T = template(Config), + {ok, A} = py_session:new(T), + {ok, B} = py_session:new(T), + {error, {child_exited, {signal, 6}}} = py_context:call(A, ?MOD, abort, []), + {error, {child_exited, {signal, 6}}} = py_context:eval(A, <<"1">>), + true = is_process_alive(A), + {ok, 1} = py_context:eval(B, <<"1">>), + {ok, C} = py_session:new(T), + {ok, 1} = py_context:eval(C, <<"1">>), + MA = erlang:monitor(process, A), + [py_session:close(X) || X <- [A, B, C]], + %% close/1 replies first, then the context removes its directory and exits + receive {'DOWN', MA, process, A, _} -> ok after 5000 -> ct:fail(session_not_stopped) end, + ok. + +test_kill_session(Config) -> + T = template(Config), + {ok, S} = py_session:new(T), + {ok, Pid} = py_context:call(S, ?MOD, pid, []), + ok = py_context:kill(S), + {error, killed} = py_context:eval(S, <<"1">>), + ok = wait_gone(Pid), + py_session:close(S). + +%% close/1 kills the child (reaped, no zombie) and removes its directory. +test_close_reaps_and_cleans(Config) -> + T = template(Config), + {ok, S} = py_session:new(T), + {ok, Pid} = py_context:call(S, ?MOD, pid, []), + {ok, Cwd} = py_context:call(S, ?MOD, cwd, []), + true = filelib:is_dir(Cwd), + ok = py_session:close(S), + ok = wait_gone(Pid), + ok = wait_no_dir(Cwd, 100), + ok. + +%% removed by the context process right after close/1 returns +wait_no_dir(Dir, 0) -> + ct:fail({still_there, Dir}); +wait_no_dir(Dir, N) -> + case filelib:is_dir(Dir) of + false -> ok; + true -> timer:sleep(20), wait_no_dir(Dir, N - 1) + end. + +%% A session is linked to the process that created it. +test_caller_crash_stops_session(Config) -> + T = template(Config), + Self = self(), + {Owner, Mon} = spawn_monitor(fun() -> + {ok, S} = py_session:new(T), + {ok, Pid} = py_context:call(S, ?MOD, pid, []), + Self ! {session, S, Pid}, + receive never -> ok end + end), + {S, Pid} = receive {session, S0, P0} -> {S0, P0} after 10000 -> ct:fail(no_session) end, + SMon = erlang:monitor(process, S), + exit(Owner, crash), + receive {'DOWN', Mon, process, Owner, crash} -> ok end, + receive {'DOWN', SMon, process, S, _} -> ok after 5000 -> ct:fail(session_survived) end, + ok = wait_gone(Pid). + +test_run_helper(Config) -> + T = template(Config), + {ok, 42} = py_session:run(T, builtins, int, [<<"42">>]), + {ok, none} = py_session:run(T, ?MOD, peek, []), + {error, {'ValueError', _}} = py_session:run(T, builtins, int, [<<"x">>]), + %% nothing left behind + #{sessions := 0} = wait_sessions(T, 0), + ok. + +%% print() in a session ends up in the Erlang logger: through the port for a +%% spawned child, as log events for a forked one (whose stdio is detached). +test_print_is_logged(Config) -> + #{level := Level} = logger:get_primary_config(), + ok = logger:set_primary_config(level, info), + ok = logger:add_handler(sess_capture, ?MODULE, #{config => #{pid => self()}}), + try + T = template(Config), + {ok, S} = py_session:new(T), + {ok, none} = py_context:eval(S, <<"print('hello-from-session', flush=True)">>), + ok = wait_logged(<<"hello-from-session">>, 50), + py_session:close(S) + after + logger:remove_handler(sess_capture), + logger:set_primary_config(level, Level) + end. + +log(#{msg := Msg}, #{config := #{pid := Pid}}) -> + Text = case Msg of + {string, S} -> S; + {report, R} -> io_lib:format("~p", [R]); + {Format, Args} -> io_lib:format(Format, Args) + end, + Pid ! {logged, unicode:characters_to_binary(Text)}, + ok. + +wait_logged(_Needle, 0) -> + ct:fail(not_logged); +wait_logged(Needle, N) -> + receive + {logged, Text} -> + case binary:match(Text, Needle) of + nomatch -> wait_logged(Needle, N); + _ -> ok + end + after 100 -> + wait_logged(Needle, N - 1) + end. + +test_rlimits_apply(Config) -> + T = template(Config, #{rlimits => #{nofile => 64}}), + {ok, S} = py_session:new(T), + {ok, 64} = py_context:eval(S, <<"__import__('resource').getrlimit(__import__('resource').RLIMIT_NOFILE)[0]">>), + py_session:close(S). + +%% refresh/1 prepares the template again: new sessions see the new code, +%% live ones keep what they started with. +test_refresh_picks_new_code(Config) -> + Dir = filename:join(?config(priv_dir, Config), atom_to_list(?config(start, Config))), + ok = filelib:ensure_dir(filename:join(Dir, "x")), + File = filename:join(Dir, "sess_versioned.py"), + ok = file:write_file(File, <<"VERSION = 1\n">>), + T = template(Config, #{paths => [Dir], imports => [sess_versioned]}), + {ok, Old} = py_session:new(T), + {ok, 1} = py_context:eval(Old, <<"__import__('sess_versioned').VERSION">>), + ok = file:write_file(File, <<"VERSION = 2\n">>), + _ = file:del_dir_r(filename:join(Dir, "__pycache__")), + ok = py_session:refresh(T), + {ok, New} = py_session:new(T), + {ok, 2} = py_context:eval(New, <<"__import__('sess_versioned').VERSION">>), + {ok, 1} = py_context:eval(Old, <<"__import__('sess_versioned').VERSION">>), + py_session:close(Old), + py_session:close(New). + +test_bad_template_options(Config) -> + Start = ?config(start, Config), + {error, {badarg, {start, nope}}} = py_session:template(#{start => nope}), + {error, {badarg, {hash_seed, -1}}} = py_session:template(#{start => Start, hash_seed => -1}), + {error, {python_not_found, _}} = py_session:template(#{start => Start, python => "/no/such/python"}), + case Start of + fork -> + {error, {template_failed, {init_failed, {'ModuleNotFoundError', _}}}} = + py_session:template(#{start => fork, imports => [no_such_module_xyz]}); + spawn -> + ok + end, + ok. + +%%% ============================================================================ +%%% fork only +%%% ============================================================================ + +%% A zygote that dies is rebuilt; the sessions it forked keep working. +test_zygote_crash_rebuilds(Config) -> + T = template(Config), + {ok, S} = py_session:new(T), + #{zygotes := [#{os_pid := Z1}]} = py_session:info(T), + _ = py_nif:os_kill(Z1, 9), + Z2 = wait_new_zygote(T, Z1, 50), + true = Z2 =/= Z1, + {ok, 1} = py_context:eval(S, <<"1">>), + {ok, Pid} = py_context:call(S, ?MOD, pid, []), + {ok, S2} = py_session:new(T), + {ok, 2} = py_context:eval(S2, <<"2">>), + %% the orphaned session is still reaped when closed + py_session:close(S), + ok = wait_gone(Pid), + py_session:close(S2). + +%% A thread started at import would be lost in every fork: refused. +test_thread_at_import_refused(Config) -> + {error, {template_failed, {init_failed, {threads, [<<"import-time-thread">>]}}}} = + py_session:template(#{start => fork, paths => [?config(test_dir, Config)], + imports => [py_test_session_thread]}), + ok. + +%% `erlang' can be imported by template code, but not called before a +%% session exists. +test_erlang_call_during_preload(Config) -> + T = template(Config, #{preload => + <<"import py_test_session\nPRELOAD_CALL = py_test_session.call_during_preload()">>}), + {ok, S} = py_session:new(T), + {ok, Msg} = py_context:eval(S, <<"PRELOAD_CALL">>), + {match, _} = re:run(Msg, <<"not connected">>), + py_session:close(S). + +%% The template reports how a child died after the pid is gone; on a busy +%% machine that can take a while. The session must wait for the report +%% rather than guess "killed" from the pid. +test_late_exit_report(Config) -> + T = template(Config), + {ok, S} = py_session:new(T), + ok = sys:suspend(T), + spawn(fun() -> timer:sleep(700), sys:resume(T) end), + {error, {child_exited, {signal, 6}}} = py_context:call(S, ?MOD, abort, []), + py_session:close(S). + +test_info_counts_forks(Config) -> + T = template(Config, #{zygotes => 2}), + #{start := fork, zygotes := [_, _], forks := 0} = py_session:info(T), + Ss = [begin {ok, S} = py_session:new(T), S end || _ <- lists:seq(1, 4)], + #{forks := 4, sessions := 4} = py_session:info(T), + [py_session:close(S) || S <- Ss], + #{sessions := 0} = wait_sessions(T, 0), + ok. + +%%% ============================================================================ +%%% spawn only +%%% ============================================================================ + +test_warm_pool(Config) -> + T = template(Config, #{warm => 2}), + #{warm := 2} = wait_warm(T, 2, 100), + {Us, {ok, S}} = timer:tc(fun() -> py_session:new(T) end), + ct:pal("session from the warm pool in ~p us", [Us]), + {links, Links} = process_info(S, links), + true = lists:member(self(), Links), + false = lists:member(T, Links), + {ok, 1} = py_context:eval(S, <<"1">>), + #{warm := 2} = wait_warm(T, 2, 100), + py_session:close(S). + +%% spawn starts each session from scratch, so import-time threads are fine. +test_thread_at_import_spawned(Config) -> + T = template(Config, #{imports => [py_test_session_thread]}), + {ok, S} = py_session:new(T), + {ok, 2} = py_context:call(S, ?MOD, thread_count, []), + py_session:close(S). + +%%% ============================================================================ +%%% reimport +%%% ============================================================================ + +-define(RMOD, py_test_reimport). + +%% Each run imports the module again: its globals start fresh every time, +%% and the interpreter's own copy is never touched. +test_reimport_runs_are_isolated(Config) -> + T = rtemplate(Config), + [{ok, 1} = py_session:run(T, ?RMOD, bump, []) || _ <- lists:seq(1, 20)], + {ok, <<"py_test_reimport">>} = py_session:run(T, ?RMOD, whoami, []), + %% the context itself never imported it + #{contexts := [C | _]} = py_session:info(T), + {ok, false} = py_context:eval(C, <<"'py_test_reimport' in __import__('sys').modules">>), + ok. + +%% What is shared on purpose: the standard library and `imports'. +test_reimport_shares_stdlib_and_imports(Config) -> + T = rtemplate(Config, #{contexts => 1, imports => [py_test_reimport_shared]}), + {ok, 1} = py_session:run(T, ?RMOD, bump_shared, []), + {ok, 2} = py_session:run(T, ?RMOD, bump_shared, []), + {ok, <<"m">>} = py_session:run(T, ?RMOD, mark_stdlib, [<<"m">>]), + {ok, <<"m">>} = py_session:run(T, ?RMOD, read_stdlib_mark, []), + {ok, 1} = py_session:run(T, ?RMOD, bump, []), + ok. + +%% Contexts run sandboxes at the same time without seeing each other's. +test_reimport_concurrent_runs(Config) -> + T = rtemplate(Config, #{contexts => 4}), + Self = self(), + Pids = [spawn_link(fun() -> + Self ! {done, [py_session:run(T, ?RMOD, bump, []) || _ <- lists:seq(1, 50)]} + end) || _ <- lists:seq(1, 8)], + [receive {done, R} -> R = lists:duplicate(50, {ok, 1}) after 60000 -> ct:fail(timeout) end + || _ <- Pids], + ok. + +%% A run calls Erlang, whose callback runs another re-import run on the +%% same template (one context: the nested run lands on the waiting one). +test_reimport_reentrance(Config) -> + T = rtemplate(Config, #{contexts => 1}), + py:register_function(reimport_nested, fun([N]) -> + {ok, R} = py_session:run(T, ?RMOD, bump, [], #{timeout => 10000}), + N + R + end), + try + [{ok, 11} = py_session:run(T, ?RMOD, call_back, [10], #{timeout => 10000}) + || _ <- lists:seq(1, 10)], + {ok, 1} = py_session:run(T, ?RMOD, bump, []) + after + py:unregister_function(reimport_nested) + end. + +%% Python code that reads sys.modules (typing) finds the run's module. C +%% code that reads the interpreter's own table (the C pickle) does not: +%% a class to pickle belongs in a shared module. +test_reimport_typing_and_pickle(Config) -> + T = rtemplate(Config, #{imports => [py_test_reimport_shared]}), + {ok, [<<"x">>, <<"y">>]} = py_session:run(T, ?RMOD, typed, []), + {ok, 3} = py_session:run(T, ?RMOD, pickle_shared, []), + %% KeyError on 3.14, PicklingError on 3.11: either way it fails + {ok, Err} = py_session:run(T, ?RMOD, pickle_reimported, []), + true = lists:member(Err, [<<"KeyError">>, <<"PicklingError">>]), + ok. + +test_reimport_errors(Config) -> + T = rtemplate(Config), + {error, {'ValueError', _}} = py_session:run(T, ?RMOD, fail, [<<"boom">>]), + {error, _} = py_session:run(T, no_such_module_xyz, f, []), + {ok, 1} = py_session:run(T, ?RMOD, bump, []), + ok. + +test_reimport_no_session_process(Config) -> + T = rtemplate(Config), + {error, {not_supported, reimport}} = py_session:new(T), + ok. + +test_reimport_refresh(Config) -> + T = rtemplate(Config, #{contexts => 2}), + #{contexts := Old} = py_session:info(T), + ok = py_session:refresh(T), + #{contexts := New} = py_session:info(T), + [] = [C || C <- New, lists:member(C, Old)], + {ok, 1} = py_session:run(T, ?RMOD, bump, []), + ok. + +test_reimport_bad_options(Config) -> + {error, {badarg, {mode, isolated}}} = + py_session:template(#{start => reimport, mode => isolated}), + {error, {badarg, {contexts, 0}}} = + py_session:template(#{start => reimport, mode => ?config(mode, Config), contexts => 0}), + ok. + +rtemplate(Config) -> + rtemplate(Config, #{}). + +rtemplate(Config, Extra) -> + {ok, T} = py_session:template(maps:merge(#{start => reimport, mode => ?config(mode, Config), + contexts => 2, + paths => [?config(test_dir, Config)]}, Extra)), + T. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +template(Config) -> + template(Config, #{}). + +template(Config, Extra) -> + TestDir = ?config(test_dir, Config), + Paths = [TestDir | maps:get(paths, Extra, [])], + Imports = [?MOD | maps:get(imports, Extra, [])], + Opts = maps:merge(#{start => ?config(start, Config)}, + Extra#{paths => Paths, imports => Imports}), + {ok, T} = py_session:template(Opts), + T. + +wait_gone(Pid) -> + wait_gone(Pid, 100). + +wait_gone(Pid, 0) -> + ct:fail({still_alive, Pid}); +wait_gone(Pid, N) -> + case py_child:os_pid_alive(Pid) of + false -> ok; + true -> timer:sleep(20), wait_gone(Pid, N - 1) + end. + +wait_new_zygote(T, Old, 0) -> + ct:fail({zygote_not_rebuilt, Old, py_session:info(T)}); +wait_new_zygote(T, Old, N) -> + case py_session:info(T) of + #{zygotes := [#{os_pid := P}]} when P =/= Old -> P; + _ -> timer:sleep(50), wait_new_zygote(T, Old, N - 1) + end. + +wait_sessions(T, Want) -> + wait_sessions(T, Want, 100). + +wait_sessions(T, _Want, 0) -> + py_session:info(T); +wait_sessions(T, Want, N) -> + case py_session:info(T) of + #{sessions := Want} = I -> I; + _ -> timer:sleep(20), wait_sessions(T, Want, N - 1) + end. + +wait_warm(T, _Want, 0) -> + py_session:info(T); +wait_warm(T, Want, N) -> + case py_session:info(T) of + #{warm := Want} = I -> I; + _ -> timer:sleep(50), wait_warm(T, Want, N - 1) + end. diff --git a/test/py_session_soak_SUITE.erl b/test/py_session_soak_SUITE.erl new file mode 100644 index 0000000..665436e --- /dev/null +++ b/test/py_session_soak_SUITE.erl @@ -0,0 +1,144 @@ +%%% @doc Soak test for sessions: many callers opening, using, crashing, +%%% killing and closing sessions from fork and spawn templates for a while, +%%% then resource counters checked against their baseline. It shows that +%%% every operation returns (no deadlock) and that nothing leaks: Erlang +%%% processes, ports, VM file descriptors, session children, zombies and +%%% scratch directories. +%%% +%%% Duration is 30 s by default; set `PY_SESSION_SOAK_SECONDS' to change it. +-module(py_session_soak_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, init_per_suite/1, end_per_suite/1]). +-export([test_session_churn_no_leak/1]). + +-define(MOD, py_test_session). + +all() -> [test_session_churn_no_leak]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + py:register_function(sess_reenter, fun([Ctx, N]) -> + {ok, R} = py_context:call(Ctx, ?MOD, reenter, [N - 1], #{}, 20000), + R + end), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + py:unregister_function(sess_reenter), + ok. + +test_session_churn_no_leak(Config) -> + Seconds = case os:getenv("PY_SESSION_SOAK_SECONDS") of + false -> 30; + S -> list_to_integer(S) + end, + TestDir = ?config(test_dir, Config), + {ok, Fork} = py_session:template(#{start => fork, zygotes => 2, paths => [TestDir], + imports => [?MOD]}), + {ok, Spawn} = py_session:template(#{start => spawn, warm => 2, paths => [TestDir], + imports => [?MOD]}), + timer:sleep(1000), + Base = counters(), + ct:print("baseline: ~p", [Base]), + Deadline = erlang:monotonic_time(millisecond) + Seconds * 1000, + Self = self(), + Workers = [spawn_link(fun() -> + rand:seed(exsss, {I, I * 7, I * 13}), + T = case I rem 4 of 0 -> Spawn; _ -> Fork end, + Self ! {done, self(), churn(T, Deadline, 0, [])} + end) || I <- lists:seq(1, 12)], + Results = [receive {done, W, R} -> R after (Seconds + 120) * 1000 -> ct:fail(worker_hung) end + || W <- Workers], + Ops = lists:sum([N || {N, _} <- Results]), + Errs = lists:append([E || {_, E} <- Results]), + ct:print("soak: ~p sessions in ~p s, ~p unexpected results: ~p", + [Ops, Seconds, length(Errs), lists:sublist(Errs, 5)]), + [] = Errs, + true = Ops > 0, + #{sessions := 0} = wait_idle(Fork, 100), + py_session:stop_template(Fork), + py_session:stop_template(Spawn), + timer:sleep(1000), + After = counters(), + ct:print("after: ~p", [After]), + check_no_growth(Base, After). + +%% One session per round, with a random use of it +churn(T, Deadline, N, Errs) -> + case erlang:monotonic_time(millisecond) < Deadline of + false -> + {N, Errs}; + true -> + {ok, S} = py_session:new(T), + Got = use(rand:uniform(8), S), + ok = py_session:close(S), + churn(T, Deadline, N + 1, case Got of ok -> Errs; Bad -> [Bad | Errs] end) + end. + +use(1, S) -> expect({ok, 3}, py_context:call(S, ?MOD, reenter, [3])); +use(2, S) -> expect({error, {child_exited, {signal, 6}}}, py_context:call(S, ?MOD, abort, [])); +use(3, S) -> expect(ok, py_context:kill(S)); +use(4, S) -> expect({error, timeout}, py_context:call(S, ?MOD, sleep, [10], #{}, 50)); +use(5, S) -> + %% a thread left running and a file written, then closed + _ = py_context:call(S, ?MOD, start_thread, []), + expect({ok, [<<"f">>]}, py_context:call(S, ?MOD, write_file, [<<"f">>, <<"x">>])); +use(6, S) -> expect({ok, none}, py_context:call(S, ?MOD, peek, [])); +use(7, S) -> expect(ok, py_context:exec(S, <<"import json; json.dumps(list(range(1000)))">>)); +use(8, _S) -> ok. + +expect(Want, Want) -> ok; +expect(Want, Got) -> {Want, Got}. + +wait_idle(T, 0) -> + py_session:info(T); +wait_idle(T, N) -> + case py_session:info(T) of + #{sessions := 0} = I -> I; + _ -> timer:sleep(50), wait_idle(T, N - 1) + end. + +%%% ============================================================================ +%%% Counters +%%% ============================================================================ + +counters() -> + erlang:garbage_collect(), + #{processes => erlang:system_info(process_count), + ports => erlang:system_info(port_count), + refs => ets:info(py_context_refs, size), + fds => beam_fd_count(), + children => session_children(), + zombies => zombies(), + scratch => length(filelib:wildcard(filename:join(py_child:sock_dir(), "sess_*")))}. + +check_no_growth(Base, After) -> + [begin + B = maps:get(K, Base), A = maps:get(K, After), + A =< B + Slack orelse ct:fail({leak, K, B, A}) + end || {K, Slack} <- [{processes, 5}, {ports, 0}, {refs, 0}, {children, 0}, + {zombies, 0}, {scratch, 0}, {fds, 8}]], + ok. + +beam_fd_count() -> + case os:type() of + {unix, linux} -> + length(filelib:wildcard("/proc/" ++ os:getpid() ++ "/fd/*")); + _ -> + Out = os:cmd("lsof -p " ++ os:getpid() ++ " 2>/dev/null | wc -l"), + list_to_integer(string:trim(Out)) - 1 + end. + +%% Python processes started by this node: spawned children, zygotes and the +%% sessions they forked (their command line is the zygote's) +session_children() -> + Out = os:cmd("ps -ax -o command= 2>/dev/null | grep -E 'py_zygote.py|py_isolated_child.py' " + "| grep erlang_python_" ++ os:getpid() ++ " | grep -v grep | wc -l"), + list_to_integer(string:trim(Out)). + +zombies() -> + Out = os:cmd("ps -ax -o stat=,command= 2>/dev/null | grep -E '^Z' " + "| grep -E 'python|py_zygote' | grep -v grep | wc -l"), + list_to_integer(string:trim(Out)). diff --git a/test/py_session_stress_SUITE.erl b/test/py_session_stress_SUITE.erl new file mode 100644 index 0000000..b8e27e6 --- /dev/null +++ b/test/py_session_stress_SUITE.erl @@ -0,0 +1,202 @@ +%%% @doc Stress and profiling for sessions. +%%% +%%% Numbers are logged (ct:print), not asserted tightly: they show what a +%%% fresh session costs next to a plain isolated context on the same +%%% machine. The asserts only catch regressions of an order of magnitude. +-module(py_session_stress_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, init_per_suite/1, end_per_suite/1, end_per_testcase/2]). + +-export([ + test_new_latency/1, + test_new_breakdown/1, + test_throughput/1, + test_call_overhead/1, + test_memory_per_session/1 +]). + +-define(MOD, py_test_session). + +all() -> [ + test_new_latency, + test_new_breakdown, + test_throughput, + test_call_overhead, + test_memory_per_session +]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + [{test_dir, filename:join(code:lib_dir(erlang_python), "test")} | Config]. + +end_per_suite(_Config) -> + ok. + +end_per_testcase(_Case, _Config) -> + [py_session:stop_template(T) || {_, T, _, _} <- supervisor:which_children(py_session_sup)], + ok. + +%% @doc Time to a usable session (new + first call), 100 in a row, for each +%% way of starting one, next to a plain isolated context. +test_new_latency(Config) -> + Fork = template(Config, #{start => fork}), + Spawn = template(Config, #{start => spawn}), + Warm = template(Config, #{start => spawn, warm => 4}), + N = 100, + Session = fun(T) -> + fun() -> + {Us, S} = us(fun() -> + {ok, S0} = py_session:new(T), + {ok, _} = py_context:call(S0, ?MOD, peek, []), + S0 + end), + py_session:close(S), + Us + end + end, + Plain = fun() -> + {Us, C} = us(fun() -> + {ok, C0} = py_context:new(#{mode => isolated, paths => [?config(test_dir, Config)]}), + {ok, _} = py_context:call(C0, ?MOD, peek, []), + C0 + end), + py_context:stop(C), + Us + end, + Rows = [{fork, run(Session(Fork), N)}, + {spawn, run(Session(Spawn), 20)}, + %% warm: paced so the pool refills between sessions + {warm_spawn, [begin timer:sleep(80), (Session(Warm))() end || _ <- lists:seq(1, 20)]}, + {plain_isolated, run(Plain, 20)}], + [ct:print("new+call ~-15s p50 ~7.2f ms p99 ~7.2f ms", [K, p(L, 50), p(L, 99)]) || {K, L} <- Rows], + ForkP50 = p(proplists:get_value(fork, Rows), 50), + SpawnP50 = p(proplists:get_value(spawn, Rows), 50), + true = ForkP50 < SpawnP50, + ok. + +%% @doc Where a forked session's time goes: the fork request, the child +%% connecting, and its `ready' frame. Same split as the prototype. +test_new_breakdown(Config) -> + T = template(Config, #{start => fork}), + Samples = [breakdown(T) || _ <- lists:seq(1, 100)], + [ct:print("fork breakdown ~-8s p50 ~6.3f ms p99 ~6.3f ms", + [K, p([maps:get(K, S) || S <- Samples], 50), p([maps:get(K, S) || S <- Samples], 99)]) + || K <- [fork, accept, ready, total]], + ok. + +breakdown(T) -> + Path = py_child:new_sock_path("bench_"), + {ok, L} = py_child:listen(Path), + T0 = erlang:monotonic_time(microsecond), + {ok, OsPid} = py_session_template:fork(T, Path, #{}, 5000), + T1 = erlang:monotonic_time(microsecond), + {ok, S} = py_child:accept(L, {os_pid, OsPid}, 5000), + T2 = erlang:monotonic_time(microsecond), + {ok, <<_:64/native, Len:32/native>>} = socket:recv(S, 12, 5000), + {ok, _Ready} = socket:recv(S, Len, 5000), + T3 = erlang:monotonic_time(microsecond), + socket:close(S), socket:close(L), file:delete(Path), + py_child:kill_os_pid(OsPid), + #{fork => T1 - T0, accept => T2 - T1, ready => T3 - T2, total => T3 - T0}. + +%% @doc Sessions per second (new + call + close) with 1, 8 and 64 callers, +%% for one and two zygotes. +test_throughput(Config) -> + Secs = 3, + Rows = [begin + T = template(Config, #{start => fork, zygotes => Z}), + R = [{C, throughput(T, C, Secs)} || C <- [1, 8, 64]], + py_session:stop_template(T), + {Z, R} + end || Z <- [1, 2]], + [[ct:print("zygotes ~w callers ~2w: ~7.1f sessions/s p50 ~6.2f ms p99 ~7.2f ms", + [Z, C, length(L) / Secs, p(L, 50), p(L, 99)]) || {C, L} <- R] || {Z, R} <- Rows], + ok. + +throughput(T, Callers, Secs) -> + Self = self(), + Deadline = erlang:monotonic_time(millisecond) + Secs * 1000, + Loop = fun L(Acc) -> + case erlang:monotonic_time(millisecond) < Deadline of + true -> + {Us, ok} = us(fun() -> + {ok, S} = py_session:new(T), + {ok, _} = py_context:call(S, ?MOD, peek, []), + py_session:close(S) + end), + L([Us | Acc]); + false -> + Acc + end + end, + Pids = [spawn_link(fun() -> Self ! {done, self(), Loop([])} end) || _ <- lists:seq(1, Callers)], + lists:append([receive {done, P, L} -> L after 60000 -> ct:fail(caller_hung) end || P <- Pids]). + +%% @doc A call in a session costs what it costs in an isolated context. +test_call_overhead(Config) -> + T = template(Config, #{start => fork}), + {ok, S} = py_session:new(T), + {ok, C} = py_context:new(#{mode => isolated, paths => [?config(test_dir, Config)]}), + Call = fun(Ctx) -> fun() -> element(1, us(fun() -> {ok, _} = py_context:call(Ctx, ?MOD, peek, []) end)) end end, + Reenter = fun(Ctx) -> fun() -> element(1, us(fun() -> {ok, 1} = py_context:call(Ctx, ?MOD, reenter, [1]) end)) end end, + py:register_function(sess_reenter, fun([Ctx, N]) -> + {ok, R} = py_context:call(Ctx, ?MOD, reenter, [N - 1]), R + end), + try + Rows = [{session_call, run(Call(S), 5000)}, + {isolated_call, run(Call(C), 5000)}, + {session_reenter, run(Reenter(S), 2000)}, + {isolated_reenter, run(Reenter(C), 2000)}], + [ct:print("~-17s p50 ~6.1f us p99 ~6.1f us", [K, p(L, 50) * 1000, p(L, 99) * 1000]) + || {K, L} <- Rows], + SessionP50 = p(proplists:get_value(session_call, Rows), 50), + PlainP50 = p(proplists:get_value(isolated_call, Rows), 50), + true = SessionP50 < PlainP50 * 2 + after + py:unregister_function(sess_reenter), + py_session:close(S), + py_context:stop(C) + end. + +%% @doc Resident memory of 20 live sessions, forked vs spawned. A forked +%% session shares the zygote's pages until it writes them. +test_memory_per_session(Config) -> + Rows = [begin + T = template(Config, #{start => Start}), + Ss = [begin {ok, S} = py_session:new(T), S end || _ <- lists:seq(1, 20)], + Pids = [begin {ok, P} = py_context:call(S, ?MOD, pid, []), P end || S <- Ss], + Rss = [rss_kb(P) || P <- Pids], + [py_session:close(S) || S <- Ss], + {Start, Rss} + end || Start <- [fork, spawn]], + [ct:print("~-5s rss per session p50 ~7.1f MB (ps rss counts shared pages in full)", + [K, lists:nth(length(L) div 2 + 1, lists:sort(L)) / 1024]) || {K, L} <- Rows], + ok. + +%%% ============================================================================ +%%% Helpers +%%% ============================================================================ + +template(Config, Opts) -> + {ok, T} = py_session:template(Opts#{paths => [?config(test_dir, Config)], + imports => [?MOD]}), + T. + +run(F, N) -> + _ = [F() || _ <- lists:seq(1, 3)], + [F() || _ <- lists:seq(1, N)]. + +us(F) -> + T0 = erlang:monotonic_time(microsecond), + R = F(), + {erlang:monotonic_time(microsecond) - T0, R}. + +%% Percentile in milliseconds +p(L, P) -> + S = lists:sort(L), + lists:nth(max(1, round(P / 100 * length(S))), S) / 1000. + +rss_kb(OsPid) -> + list_to_integer(string:trim(os:cmd("ps -o rss= -p " ++ integer_to_list(OsPid)))). diff --git a/test/py_test_reimport.py b/test/py_test_reimport.py new file mode 100644 index 0000000..66a506c --- /dev/null +++ b/test/py_test_reimport.py @@ -0,0 +1,69 @@ +"""Imported again in every re-import run (py_session_SUITE reimport groups).""" + +import json +import pickle +import typing +from dataclasses import dataclass + +import erlang +import py_test_reimport_shared as shared + +COUNT = {'n': 0} + + +@dataclass +class Point: + x: int + y: int + + +def bump(): + COUNT['n'] += 1 + return COUNT['n'] + + +def bump_shared(): + return shared.bump() + + +def mark_stdlib(value): + """The standard library is shared: this is seen by later runs.""" + json._reimport_mark = value + return value + + +def read_stdlib_mark(): + return getattr(json, '_reimport_mark', None) + + +def call_back(n): + return erlang.call('reimport_nested', n) + + +def typed(): + """typing reads sys.modules from Python: it finds the run's module.""" + return sorted(typing.get_type_hints(Point)) + + +def pickle_shared(): + """A class from a shared module pickles.""" + p = pickle.loads(pickle.dumps(shared.Pair(1, 2))) + return p.a + p.b + + +def pickle_reimported(): + """The C pickle looks the class's module up in the interpreter's own + table, where a re-imported module is not.""" + try: + pickle.dumps(Point(1, 2)) + except Exception as exc: + return type(exc).__name__ + return 'pickled' + + +def fail(msg): + raise ValueError(msg) + + +def whoami(): + return __name__ diff --git a/test/py_test_reimport_shared.py b/test/py_test_reimport_shared.py new file mode 100644 index 0000000..fc2f648 --- /dev/null +++ b/test/py_test_reimport_shared.py @@ -0,0 +1,16 @@ +"""Listed in a reimport template's `imports`: shared by every run.""" + +from dataclasses import dataclass + +COUNT = {'n': 0} + + +@dataclass +class Pair: + a: int + b: int + + +def bump(): + COUNT['n'] += 1 + return COUNT['n'] diff --git a/test/py_test_session.py b/test/py_test_session.py new file mode 100644 index 0000000..f031e54 --- /dev/null +++ b/test/py_test_session.py @@ -0,0 +1,133 @@ +"""Helpers for py_session_SUITE. Imported by the template, so everything at +module level here runs once, in the zygote (start => fork) or in each child +(start => spawn).""" + +import os +import random +import sys +import threading +import time + +import erlang +from erlang import call # resolved before any session exists + +STATE = {} +IMPORTED_IN = os.getpid() + + +def mark(value): + STATE['mark'] = value + return value + + +def peek(): + return STATE.get('mark') + + +def pid(): + return os.getpid() + + +def cwd(): + return os.getcwd() + + +def env_names(): + return sorted(os.environ) + + +def env_get(name): + return os.environ.get(name) + + +def env_set(name, value): + os.environ[name] = value + return value + + +def put_module(name): + import types + sys.modules[name] = types.ModuleType(name) + return name in sys.modules + + +def has_module(name): + return name in sys.modules + + +def start_thread(): + threading.Thread(target=time.sleep, args=(60,), daemon=True).start() + return threading.active_count() + + +def thread_count(): + return threading.active_count() + + +def write_file(name, data): + with open(name, 'w') as f: + f.write(data) + return sorted(os.listdir('.')) + + +def list_cwd(): + return sorted(os.listdir('.')) + + +def random_value(): + return random.random() + + +def hash_probe(): + return (hash('erlang-python'), repr(list({'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'}))) + + +def reenter(n): + """Python -> Erlang -> this same session -> ... n levels deep.""" + return 0 if n == 0 else call('sess_reenter', erlang.self(), n) + 1 + + +def cross(n): + """Calls into another session through Erlang.""" + return call('sess_cross', n) + + +def nested_sleep_then(seconds): + """Erlang runs sleep() in this session with a short timeout: the nested + request is interrupted, this outer one carries on.""" + inner = call('sess_nested_sleep', erlang.self(), seconds) + return ('outer-done', inner) + + +def sleep(seconds): + time.sleep(seconds) + return 'slept' + + +async def async_add(a, b): + return a + b + + +def fd_is_open(fd): + os.fstat(fd) + return True + + +def imported_in(): + return IMPORTED_IN + + +def call_during_preload(): + try: + call('anything') + except RuntimeError as exc: + return str(exc) + return 'no error' + + +def abort(): + os.abort() + + +def open_files(): + return len(os.listdir('/dev/fd')) diff --git a/test/py_test_session_thread.py b/test/py_test_session_thread.py new file mode 100644 index 0000000..45ca81c --- /dev/null +++ b/test/py_test_session_thread.py @@ -0,0 +1,6 @@ +"""Starts a thread at import time: a template importing it cannot be forked.""" + +import threading +import time + +threading.Thread(target=time.sleep, args=(60,), name='import-time-thread', daemon=True).start()