Chrome pool redo: reuses a single (headless) chrome for all spaces in the server, not one per instance
This commit is contained in:
+132
-116
@@ -1,43 +1,28 @@
|
||||
use std::path::Path;
|
||||
|
||||
/// Resolved configuration for launching the headless browser.
|
||||
/// Configuration for the single, server-wide headless browser. Resolved once at
|
||||
/// startup and shared by every space's page.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ChromeConfig {
|
||||
pub chrome_path: String,
|
||||
pub server_url: String,
|
||||
/// Headless auth token, always appended to the page URL as `&token=…`. When
|
||||
/// authentication is disabled the open server ignores it; when enabled, the
|
||||
/// headless-token authorizer accepts it so the headless page is authorized.
|
||||
pub headless_token: String,
|
||||
pub user_data_dir: String,
|
||||
pub show: bool,
|
||||
/// Forward the headless page's captured `console.*` output to the server's
|
||||
/// `tracing` log (under the `runtime_console` target) in addition to the
|
||||
/// `/.runtime/logs` buffer. On by default; disabled with
|
||||
/// `SB_CHROME_LOG_CONSOLE=0`.
|
||||
pub log_console: bool,
|
||||
}
|
||||
|
||||
impl ChromeConfig {
|
||||
/// Build from the process environment. `server_url` is the loopback base
|
||||
/// URL the headless page should load (`http://127.0.0.1:<port><prefix>`);
|
||||
/// `headless_token` is always generated by the caller; it's only consulted
|
||||
/// by the auth layer when authentication is enabled.
|
||||
pub fn from_env(
|
||||
server_url: String,
|
||||
headless_token: String,
|
||||
space_folder: &str,
|
||||
read_only: bool,
|
||||
) -> Option<Self> {
|
||||
/// Build from the process environment. `server_root` is the server's root
|
||||
/// directory, used for the default profile location. Returns `None` when
|
||||
/// the runtime API is disabled or no Chrome can be found.
|
||||
pub fn from_env(server_root: &Path) -> Option<Self> {
|
||||
let env = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());
|
||||
let runtime_api_enabled =
|
||||
!matches!(env("SB_RUNTIME_API").as_deref(), Some("0") | Some("false"));
|
||||
Self::resolve(
|
||||
env("SB_CHROME_PATH"),
|
||||
env("CHROMIUM_PATH"),
|
||||
server_url,
|
||||
headless_token,
|
||||
space_folder.to_string(),
|
||||
env("SB_CHROME_DATA_DIR"),
|
||||
server_root,
|
||||
env("SB_CHROME_SHOW").is_some(),
|
||||
// On by default; disabled only with SB_CHROME_LOG_CONSOLE=0/false
|
||||
// (matches the SB_RUNTIME_API opt-out convention).
|
||||
@@ -45,42 +30,35 @@ impl ChromeConfig {
|
||||
env("SB_CHROME_LOG_CONSOLE").as_deref(),
|
||||
Some("0") | Some("false")
|
||||
),
|
||||
read_only,
|
||||
runtime_api_enabled,
|
||||
)
|
||||
}
|
||||
|
||||
/// Pure resolution (unit-tested). Returns `None` when the runtime API is
|
||||
/// disabled, the space is read-only, or no Chrome can be found.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Pure resolution (unit-tested). Seven parameters is exactly clippy's
|
||||
/// `too_many_arguments` threshold, so no `allow` is needed — do not add one.
|
||||
pub fn resolve(
|
||||
sb_chrome_path: Option<String>,
|
||||
chromium_path: Option<String>,
|
||||
server_url: String,
|
||||
headless_token: String,
|
||||
space_folder: String,
|
||||
chrome_data_dir: Option<String>,
|
||||
server_root: &Path,
|
||||
show: bool,
|
||||
log_console: bool,
|
||||
read_only: bool,
|
||||
runtime_api_enabled: bool,
|
||||
) -> Option<Self> {
|
||||
if !runtime_api_enabled || read_only {
|
||||
if !runtime_api_enabled {
|
||||
return None;
|
||||
}
|
||||
let chrome_path = sb_chrome_path.or(chromium_path).or_else(find_chrome)?;
|
||||
let user_data_dir = std::env::var("SB_CHROME_DATA_DIR")
|
||||
.ok()
|
||||
let user_data_dir = chrome_data_dir
|
||||
.filter(|v| !v.is_empty())
|
||||
.unwrap_or_else(|| {
|
||||
Path::new(&space_folder)
|
||||
server_root
|
||||
.join(".chrome-data")
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
});
|
||||
Some(Self {
|
||||
chrome_path,
|
||||
server_url,
|
||||
headless_token,
|
||||
user_data_dir,
|
||||
show,
|
||||
log_console,
|
||||
@@ -88,6 +66,48 @@ impl ChromeConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-space page configuration: where this space's client lives and how its
|
||||
/// headless page authenticates to it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SpacePage {
|
||||
/// Loopback base URL for this space (`http://127.0.0.1:<port><prefix>`).
|
||||
pub server_url: String,
|
||||
/// Headless auth token, seeded as an HTTP-only session cookie before the
|
||||
/// page navigates. Never appears in the URL.
|
||||
pub headless_token: String,
|
||||
/// This space's cookie name (`silverbullet_headless_<space_id>`).
|
||||
pub cookie_name: String,
|
||||
}
|
||||
|
||||
impl SpacePage {
|
||||
/// The headless page URL: the space base URL with a trailing slash and
|
||||
/// `?headless=1`. Authentication rides in the cookie, not the query string.
|
||||
pub fn page_url(&self) -> String {
|
||||
let base = self.server_url.trim_end_matches('/');
|
||||
format!("{base}/?headless=1")
|
||||
}
|
||||
|
||||
/// `Path` attribute for this space's auth cookie: the space prefix, or `/`
|
||||
/// for a root-bound space.
|
||||
pub fn cookie_path(&self) -> String {
|
||||
let path = self
|
||||
.server_url
|
||||
.split_once("://")
|
||||
.and_then(|(_, authority_and_path)| {
|
||||
authority_and_path
|
||||
.find('/')
|
||||
.map(|index| &authority_and_path[index..])
|
||||
})
|
||||
.unwrap_or("/");
|
||||
let path = path.trim_end_matches('/');
|
||||
if path.is_empty() {
|
||||
"/".to_string()
|
||||
} else {
|
||||
path.to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a Chrome/Chromium executable from platform-specific candidates.
|
||||
pub fn find_chrome() -> Option<String> {
|
||||
if cfg!(target_os = "macos") {
|
||||
@@ -167,100 +187,96 @@ fn which_on_path(name: &str) -> Option<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn explicit_path_wins_over_discovery() {
|
||||
let cfg = ChromeConfig::resolve(
|
||||
Some("/custom/chrome".into()),
|
||||
fn resolve(root: &str, chrome_data_dir: Option<&str>) -> Option<ChromeConfig> {
|
||||
ChromeConfig::resolve(
|
||||
Some("/bin/chrome".into()),
|
||||
None,
|
||||
"http://127.0.0.1:3000".into(),
|
||||
String::new(),
|
||||
"/tmp/space".into(),
|
||||
false,
|
||||
false,
|
||||
chrome_data_dir.map(str::to_string),
|
||||
Path::new(root),
|
||||
false,
|
||||
true,
|
||||
);
|
||||
assert_eq!(cfg.unwrap().chrome_path, "/custom/chrome");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chromium_path_is_fallback_for_explicit() {
|
||||
let cfg = ChromeConfig::resolve(
|
||||
None,
|
||||
Some("/usr/bin/chromium".into()),
|
||||
"u".into(),
|
||||
String::new(),
|
||||
"/s".into(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
);
|
||||
assert_eq!(cfg.unwrap().chrome_path, "/usr/bin/chromium");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_when_runtime_api_off_or_read_only() {
|
||||
assert!(ChromeConfig::resolve(
|
||||
Some("/x".into()),
|
||||
None,
|
||||
"u".into(),
|
||||
String::new(),
|
||||
"/s".into(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
)
|
||||
.is_none());
|
||||
assert!(ChromeConfig::resolve(
|
||||
Some("/x".into()),
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_defaults_to_the_server_root() {
|
||||
let cfg = resolve("/srv/sb", None).expect("resolves");
|
||||
assert_eq!(cfg.chrome_path, "/bin/chrome");
|
||||
assert_eq!(cfg.user_data_dir, "/srv/sb/.chrome-data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_data_dir_wins() {
|
||||
let cfg = resolve("/srv/sb", Some("/var/cache/sb-chrome")).expect("resolves");
|
||||
assert_eq!(cfg.user_data_dir, "/var/cache/sb-chrome");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_data_dir_is_ignored() {
|
||||
let cfg = ChromeConfig::resolve(
|
||||
Some("/bin/chrome".into()),
|
||||
None,
|
||||
"u".into(),
|
||||
String::new(),
|
||||
"/s".into(),
|
||||
false,
|
||||
Some(String::new()),
|
||||
Path::new("/srv/sb"),
|
||||
false,
|
||||
true,
|
||||
true
|
||||
true,
|
||||
)
|
||||
.expect("resolves");
|
||||
assert_eq!(cfg.user_data_dir, "/srv/sb/.chrome-data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chromium_path_is_the_fallback() {
|
||||
let cfg = ChromeConfig::resolve(
|
||||
None,
|
||||
Some("/bin/chromium".into()),
|
||||
None,
|
||||
Path::new("/srv/sb"),
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
)
|
||||
.expect("resolves");
|
||||
assert_eq!(cfg.chrome_path, "/bin/chromium");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_runtime_api_resolves_to_nothing() {
|
||||
assert!(ChromeConfig::resolve(
|
||||
Some("/bin/chrome".into()),
|
||||
None,
|
||||
None,
|
||||
Path::new("/srv/sb"),
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
)
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_dir_defaults_under_space() {
|
||||
let cfg = ChromeConfig::resolve(
|
||||
Some("/x".into()),
|
||||
None,
|
||||
"u".into(),
|
||||
String::new(),
|
||||
"/space".into(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(cfg.user_data_dir.ends_with(".chrome-data"));
|
||||
assert!(cfg.user_data_dir.contains("space"));
|
||||
fn page_url_carries_no_token() {
|
||||
let page = SpacePage {
|
||||
server_url: "http://127.0.0.1:3000/notes".into(),
|
||||
headless_token: "secret".into(),
|
||||
cookie_name: "silverbullet_headless_a".into(),
|
||||
};
|
||||
assert_eq!(page.page_url(), "http://127.0.0.1:3000/notes/?headless=1");
|
||||
assert!(!page.page_url().contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_show_and_log_console_flow_through() {
|
||||
let cfg = ChromeConfig::resolve(
|
||||
Some("/x".into()),
|
||||
None,
|
||||
"u".into(),
|
||||
"tok123".into(),
|
||||
"/s".into(),
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(cfg.headless_token, "tok123");
|
||||
assert!(cfg.show);
|
||||
assert!(cfg.log_console);
|
||||
fn cookie_path_is_the_space_prefix() {
|
||||
let page = |url: &str| SpacePage {
|
||||
server_url: url.into(),
|
||||
headless_token: "secret".into(),
|
||||
cookie_name: "silverbullet_headless_a".into(),
|
||||
};
|
||||
assert_eq!(page("http://127.0.0.1:3000/notes").cookie_path(), "/notes");
|
||||
assert_eq!(page("http://127.0.0.1:3000").cookie_path(), "/");
|
||||
assert_eq!(page("http://127.0.0.1:3000/").cookie_path(), "/");
|
||||
assert_eq!(page("http://127.0.0.1:3000/a/b").cookie_path(), "/a/b");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
//! server can evaluate Space Lua and answer the objects API.
|
||||
|
||||
mod config;
|
||||
mod pool;
|
||||
mod supervisor;
|
||||
mod transport;
|
||||
|
||||
pub use config::{find_chrome, ChromeConfig};
|
||||
pub use transport::ChromeTransport;
|
||||
pub use config::{find_chrome, ChromeConfig, SpacePage};
|
||||
pub use pool::{ChromePool, SharedChromeTransport};
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
//! The shared headless-Chrome pool: one browser process for the whole server,
|
||||
//! one page per space.
|
||||
//!
|
||||
//! Launch is LAZY at two levels. The browser does not start until the first
|
||||
//! runtime request from *any* space, and a space's page is not created until
|
||||
//! that space's own first request — so Chrome never runs while the runtime API
|
||||
//! is unused, and a space nobody queries never costs a tab.
|
||||
//!
|
||||
//! `eval_js`/`wait_ready` are synchronous and block the calling thread (the
|
||||
//! server invokes them via `spawn_blocking`) they run their async work on the
|
||||
//! pool's owned runtime.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use chromiumoxide::browser::Browser;
|
||||
use chromiumoxide::page::Page;
|
||||
use serde_json::Value;
|
||||
use silverbullet_server::runtime::{ClientTransport, LogBuffer, RuntimeError};
|
||||
use tokio::runtime::Runtime;
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
|
||||
use crate::config::{ChromeConfig, SpacePage};
|
||||
use crate::supervisor::{eval_on_page, launch_browser, supervise_space};
|
||||
|
||||
/// One registered space: its live page, readiness flag, and supervisor handle.
|
||||
struct Registration {
|
||||
live: Arc<Mutex<Option<Page>>>,
|
||||
supervisor: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
/// The pool's browser slot: the live browser tagged with the generation it was
|
||||
/// launched as, or empty when there is none.
|
||||
type BrowserSlot<B> = Arc<Mutex<Option<(u64, Arc<B>)>>>;
|
||||
|
||||
/// The shared headless-Chrome pool.
|
||||
pub struct ChromePool<B = Browser> {
|
||||
rt: Option<Runtime>,
|
||||
config: ChromeConfig,
|
||||
browser: BrowserSlot<B>,
|
||||
launch_lock: Arc<Mutex<()>>,
|
||||
spaces: Arc<StdMutex<HashMap<u64, Registration>>>,
|
||||
next_id: AtomicU64,
|
||||
next_generation: AtomicU64,
|
||||
}
|
||||
|
||||
/// Clear `slot` when — and only when — it still holds `generation`. Returns
|
||||
/// whether it cleared anything.
|
||||
///
|
||||
/// This is the guard that stops a browser restart from ping-ponging: a
|
||||
/// supervisor that noticed generation *G* had died may arrive long after
|
||||
/// somebody else already launched *G+1* (a page launch takes seconds), and it
|
||||
/// must not take the live browser down with it.
|
||||
fn clear_if_generation<T>(slot: &mut Option<(u64, T)>, generation: u64) -> bool {
|
||||
match slot {
|
||||
Some((current, _)) if *current == generation => {
|
||||
*slot = None;
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
impl<B> ChromePool<B> {
|
||||
/// Construct the pool. Generic over the browser handle so tests can build a
|
||||
/// pool whose slot they are able to populate; `ChromePool::new` is the
|
||||
/// production entry point.
|
||||
fn build(config: ChromeConfig) -> Result<Arc<Self>, RuntimeError> {
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(2)
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|e| RuntimeError::Transport(format!("tokio runtime: {e}")))?;
|
||||
Ok(Arc::new(Self {
|
||||
rt: Some(rt),
|
||||
config,
|
||||
browser: Arc::new(Mutex::new(None)),
|
||||
launch_lock: Arc::new(Mutex::new(())),
|
||||
spaces: Arc::new(StdMutex::new(HashMap::new())),
|
||||
next_id: AtomicU64::new(0),
|
||||
next_generation: AtomicU64::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
/// The owned runtime. Always present until `Drop`; the `Option` exists only
|
||||
/// so `Drop` can take it.
|
||||
fn rt(&self) -> &Runtime {
|
||||
self.rt.as_ref().expect("runtime present until drop")
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &ChromeConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
/// How many spaces currently hold a transport. Registration is per
|
||||
/// *transport*, not per space id: a space being rebuilt briefly has two.
|
||||
pub fn registered_spaces(&self) -> usize {
|
||||
self.spaces.lock().unwrap_or_else(|e| e.into_inner()).len()
|
||||
}
|
||||
|
||||
/// Retire the browser of `generation` so the next `ensure_browser` launches
|
||||
/// a fresh one. Called by a supervisor that saw *this* browser fail a
|
||||
/// liveness probe.
|
||||
pub(crate) async fn discard_browser(&self, generation: u64) {
|
||||
// Taken before the slot lock so this cannot interleave with
|
||||
// `ensure_browser`'s launch-then-store.
|
||||
let _guard = self.launch_lock.lock().await;
|
||||
let mut slot = self.browser.lock().await;
|
||||
if clear_if_generation(&mut slot, generation) {
|
||||
tracing::warn!("shared headless Chrome (generation {generation}) died; retiring it");
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a browser process is currently held. Exists to pin the laziness
|
||||
/// guarantee in tests — nothing must launch Chrome before the first runtime
|
||||
/// request.
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn browser_is_launched(&self) -> bool {
|
||||
self.browser.lock().await.is_some()
|
||||
}
|
||||
|
||||
/// Install `browser` in the slot at `generation`, as a successful
|
||||
/// `ensure_browser` would. Tests only: it is the sole way to exercise
|
||||
/// `discard_browser` without launching real Chrome.
|
||||
#[cfg(test)]
|
||||
async fn seed_browser(&self, generation: u64, browser: B) {
|
||||
*self.browser.lock().await = Some((generation, Arc::new(browser)));
|
||||
}
|
||||
}
|
||||
|
||||
impl ChromePool {
|
||||
/// Create the pool. Returns an error only if the tokio runtime can't be
|
||||
/// built.
|
||||
pub fn new(config: ChromeConfig) -> Result<Arc<Self>, RuntimeError> {
|
||||
Self::build(config)
|
||||
}
|
||||
|
||||
/// Register a space and return its transport. Spawns the space's supervisor
|
||||
/// parked on its trigger; no browser or page work happens until the first
|
||||
/// runtime request.
|
||||
pub fn transport_for(
|
||||
self: &Arc<Self>,
|
||||
page: SpacePage,
|
||||
logs: LogBuffer,
|
||||
) -> SharedChromeTransport {
|
||||
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
|
||||
let live: Arc<Mutex<Option<Page>>> = Arc::new(Mutex::new(None));
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let trigger = Arc::new(Notify::new());
|
||||
|
||||
let supervisor = self.rt().spawn(supervise_space(
|
||||
self.clone(),
|
||||
page.clone(),
|
||||
live.clone(),
|
||||
ready.clone(),
|
||||
logs,
|
||||
trigger.clone(),
|
||||
));
|
||||
|
||||
self.spaces
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.insert(
|
||||
id,
|
||||
Registration {
|
||||
live: live.clone(),
|
||||
supervisor,
|
||||
},
|
||||
);
|
||||
|
||||
SharedChromeTransport {
|
||||
pool: self.clone(),
|
||||
id,
|
||||
live,
|
||||
ready,
|
||||
trigger,
|
||||
}
|
||||
}
|
||||
|
||||
/// The live browser and its generation, launching it if there isn't one.
|
||||
/// Concurrent callers serialize on `launch_lock`; the second one finds the
|
||||
/// slot already filled and returns it.
|
||||
///
|
||||
/// The returned `Arc` is a *transient* clone: use it for one `new_page` and
|
||||
/// drop it. Holding it would keep a dead Chrome's process alive across a
|
||||
/// relaunch. The generation goes back to `discard_browser` if this browser
|
||||
/// turns out to be dead.
|
||||
pub(crate) async fn ensure_browser(&self) -> Result<(u64, Arc<Browser>), String> {
|
||||
/// Cap on a single `Browser::launch`. It is awaited while holding
|
||||
/// `launch_lock`, so a Chrome that hangs on startup would otherwise
|
||||
/// block *every* space's supervisor forever.
|
||||
const LAUNCH_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
|
||||
if let Some((generation, b)) = self.browser.lock().await.as_ref() {
|
||||
return Ok((*generation, b.clone()));
|
||||
}
|
||||
let _guard = self.launch_lock.lock().await;
|
||||
if let Some((generation, b)) = self.browser.lock().await.as_ref() {
|
||||
return Ok((*generation, b.clone()));
|
||||
}
|
||||
tracing::info!(
|
||||
"runtime API used; launching shared headless Chrome ({})",
|
||||
self.config.chrome_path
|
||||
);
|
||||
let browser = match tokio::time::timeout(LAUNCH_TIMEOUT, launch_browser(&self.config)).await
|
||||
{
|
||||
Err(_) => {
|
||||
return Err(format!(
|
||||
"browser launch timed out after {}s",
|
||||
LAUNCH_TIMEOUT.as_secs()
|
||||
))
|
||||
}
|
||||
Ok(result) => Arc::new(result?),
|
||||
};
|
||||
let generation = self.next_generation.fetch_add(1, Ordering::Relaxed);
|
||||
*self.browser.lock().await = Some((generation, browser.clone()));
|
||||
Ok((generation, browser))
|
||||
}
|
||||
}
|
||||
|
||||
impl<B> Drop for ChromePool<B> {
|
||||
fn drop(&mut self) {
|
||||
// Abort every supervisor first so none of them races the teardown.
|
||||
for (_, reg) in self
|
||||
.spaces
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.drain()
|
||||
{
|
||||
reg.supervisor.abort();
|
||||
drop(reg.live);
|
||||
}
|
||||
if let Some(rt) = self.rt.take() {
|
||||
// `shutdown_background` rather than a blocking shutdown: the pool is
|
||||
// dropped from inside the server's own async context on Ctrl-C, and
|
||||
// blocking there panics.
|
||||
rt.shutdown_background();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One space's view of the shared browser: its own page, readiness flag, and
|
||||
/// lazy-launch trigger.
|
||||
pub struct SharedChromeTransport {
|
||||
pool: Arc<ChromePool>,
|
||||
id: u64,
|
||||
live: Arc<Mutex<Option<Page>>>,
|
||||
ready: Arc<AtomicBool>,
|
||||
trigger: Arc<Notify>,
|
||||
}
|
||||
|
||||
impl ClientTransport for SharedChromeTransport {
|
||||
fn eval_js(&self, js: &str, timeout: Duration) -> Result<Value, RuntimeError> {
|
||||
// Any runtime use wakes this space's supervisor, which brings the shared
|
||||
// browser up if needed and then this space's page.
|
||||
self.trigger.notify_one();
|
||||
let live = self.live.clone();
|
||||
let js = js.to_string();
|
||||
self.pool.rt().block_on(async move {
|
||||
// The timeout covers *acquiring* `live` as well as the eval itself.
|
||||
// The eval alone was never unbounded — chromiumoxide gives every
|
||||
// command a ~30s `REQUEST_TIMEOUT` of its own — but the lock wait in
|
||||
// front of it had no bound at all, and it is the part that stacks:
|
||||
// this call occupies a thread on the server's shared
|
||||
// `spawn_blocking` pool, and the supervisor's liveness probe takes
|
||||
// the same lock. Wrapping both means a caller's own timeout is the
|
||||
// real bound, instead of an implicit one from a dependency plus an
|
||||
// unbounded queue behind it.
|
||||
let attempt = async {
|
||||
let guard = live.lock().await;
|
||||
let page = guard.as_ref().ok_or(RuntimeError::NotReady)?;
|
||||
eval_on_page(page, &js).await
|
||||
};
|
||||
match tokio::time::timeout(timeout, attempt).await {
|
||||
Err(_) => Err(RuntimeError::Timeout),
|
||||
Ok(r) => r,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn wait_ready(&self, timeout: Duration) -> Result<(), RuntimeError> {
|
||||
self.trigger.notify_one();
|
||||
if self.ready.load(Ordering::Relaxed) {
|
||||
return Ok(());
|
||||
}
|
||||
let ready = self.ready.clone();
|
||||
self.pool.rt().block_on(async move {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
if ready.load(Ordering::Relaxed) {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(RuntimeError::NotReady);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn ensure_started(&self) {
|
||||
// Same lazy-launch nudge as eval/wait: a log read alone is enough to
|
||||
// bring this space's page up.
|
||||
self.trigger.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SharedChromeTransport {
|
||||
fn drop(&mut self) {
|
||||
// A space instance is rebuilt on *every* admin API space write, so
|
||||
// without this each write would leak a tab in the shared browser.
|
||||
let reg = self
|
||||
.pool
|
||||
.spaces
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.remove(&self.id);
|
||||
let Some(reg) = reg else { return };
|
||||
let Registration { live, supervisor } = reg;
|
||||
self.pool.rt().spawn(async move {
|
||||
// Abort *and wait* before touching `live`. `JoinHandle::abort` is
|
||||
// not synchronous: cancelling the supervisor while it is inside
|
||||
// `launch_page` means its page is a task local, in neither `live`
|
||||
// nor anywhere else, until its `PageCloseGuard` drops. Aborting from
|
||||
// `Drop` and closing here immediately could observe `live == None`,
|
||||
// finish, and let the supervisor then publish a page nobody will
|
||||
// ever look at again. Awaiting the handle guarantees the task (and
|
||||
// its guard) is fully torn down first.
|
||||
supervisor.abort();
|
||||
let _ = supervisor.await;
|
||||
// Two statements, so the `MutexGuard` is dropped at the `;` rather
|
||||
// than held across the `close` await — matching `supervise_space`'s
|
||||
// restart path. Nothing contends for `live` here, but the two sites
|
||||
// should read alike.
|
||||
let stale = live.lock().await.take();
|
||||
if let Some(page) = stale {
|
||||
let _ = page.close().await;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config() -> ChromeConfig {
|
||||
ChromeConfig {
|
||||
// Never launched: registration and teardown are lazy, so no real
|
||||
// browser is needed to exercise the pool's bookkeeping.
|
||||
chrome_path: "/nonexistent/chrome".into(),
|
||||
user_data_dir: "/tmp/sb-pool-test".into(),
|
||||
show: false,
|
||||
log_console: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn page(name: &str, url: &str) -> SpacePage {
|
||||
SpacePage {
|
||||
server_url: url.into(),
|
||||
headless_token: "secret".into(),
|
||||
cookie_name: format!("silverbullet_headless_{name}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registers_and_deregisters_each_space() {
|
||||
let pool = ChromePool::new(config()).unwrap();
|
||||
assert_eq!(pool.registered_spaces(), 0);
|
||||
|
||||
let a = pool.transport_for(page("a", "http://127.0.0.1:3000"), LogBuffer::new());
|
||||
let b = pool.transport_for(page("b", "http://127.0.0.1:3000/notes"), LogBuffer::new());
|
||||
assert_eq!(pool.registered_spaces(), 2);
|
||||
|
||||
drop(a);
|
||||
assert_eq!(pool.registered_spaces(), 1);
|
||||
drop(b);
|
||||
assert_eq!(pool.registered_spaces(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebuilding_a_space_does_not_deregister_its_replacement() {
|
||||
let pool = ChromePool::new(config()).unwrap();
|
||||
let old = pool.transport_for(page("a", "http://127.0.0.1:3000"), LogBuffer::new());
|
||||
let new = pool.transport_for(page("a", "http://127.0.0.1:3000"), LogBuffer::new());
|
||||
assert_eq!(pool.registered_spaces(), 2);
|
||||
|
||||
drop(old);
|
||||
assert_eq!(pool.registered_spaces(), 1);
|
||||
drop(new);
|
||||
assert_eq!(pool.registered_spaces(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_browser_is_launched_before_any_runtime_request() {
|
||||
let pool = ChromePool::new(config()).unwrap();
|
||||
assert!(!pool.browser_is_launched().await);
|
||||
|
||||
let t = pool.transport_for(page("a", "http://127.0.0.1:3000"), LogBuffer::new());
|
||||
// Give the supervisor task a chance to run before checking.
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert!(!pool.browser_is_launched().await);
|
||||
assert!(!t.is_ready());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_discard_leaves_a_newer_browser_alone() {
|
||||
let mut slot = Some((1u64, "browser-1"));
|
||||
assert!(!clear_if_generation(&mut slot, 0));
|
||||
assert_eq!(slot, Some((1, "browser-1")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn discarding_the_current_generation_clears_the_slot() {
|
||||
let mut slot = Some((1u64, "browser-1"));
|
||||
assert!(clear_if_generation(&mut slot, 1));
|
||||
assert_eq!(slot, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn discard_browser_retires_only_the_generation_it_was_given() {
|
||||
let pool = ChromePool::<&'static str>::build(config()).unwrap();
|
||||
pool.seed_browser(1, "browser-1").await;
|
||||
|
||||
// A supervisor that watched generation 0 die, arriving after somebody
|
||||
// else already launched generation 1, must leave it alone.
|
||||
pool.discard_browser(0).await;
|
||||
assert!(
|
||||
pool.browser_is_launched().await,
|
||||
"a stale generation must not retire the live browser"
|
||||
);
|
||||
|
||||
// The generation that actually failed is retired, so the next
|
||||
// `ensure_browser` launches a fresh one.
|
||||
pool.discard_browser(1).await;
|
||||
assert!(
|
||||
!pool.browser_is_launched().await,
|
||||
"the current generation must clear the slot"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn discarding_an_empty_slot_is_a_no_op() {
|
||||
let mut slot: Option<(u64, &str)> = None;
|
||||
assert!(!clear_if_generation(&mut slot, 0));
|
||||
assert_eq!(slot, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generations_are_monotonic() {
|
||||
let pool = ChromePool::new(config()).unwrap();
|
||||
let first = pool.next_generation.fetch_add(1, Ordering::Relaxed);
|
||||
let second = pool.next_generation.fetch_add(1, Ordering::Relaxed);
|
||||
assert_eq!((first, second), (0, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_failed_launch_leaves_the_slot_empty() {
|
||||
let pool = ChromePool::new(config()).unwrap();
|
||||
// `/nonexistent/chrome` cannot be spawned, so this fails fast.
|
||||
assert!(pool.ensure_browser().await.is_err());
|
||||
assert!(!pool.browser_is_launched().await);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_the_pool_inside_an_async_context_does_not_panic() {
|
||||
let pool = ChromePool::new(config()).unwrap();
|
||||
let t = pool.transport_for(page("a", "http://127.0.0.1:3000"), LogBuffer::new());
|
||||
drop(t);
|
||||
drop(pool); // must not panic
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,19 @@
|
||||
//! Browser supervisor: owns the live `Browser` + `Page`, captures console
|
||||
//! output into the shared log buffer, waits for the client runtime to signal
|
||||
//! Browser launch and per-space page supervision: brings up the shared
|
||||
//! `Browser` for the pool, opens a space's `Page` in it, captures console output
|
||||
//! into that space's log buffer, waits for the client runtime to signal
|
||||
//! readiness, and restarts (with exponential backoff) whenever the page dies.
|
||||
//!
|
||||
//! The supervisor runs as a single long-lived task on the transport's tokio
|
||||
//! runtime. It is deliberately tolerant of a not-yet-listening server: the
|
||||
//! transport is constructed *before* the server binds its port, so the first
|
||||
//! navigation may fail and is simply retried with backoff.
|
||||
//! One supervisor task per space runs on the pool's tokio runtime. It is
|
||||
//! deliberately tolerant of a not-yet-listening server: transports are
|
||||
//! constructed *before* the server binds its port, so the first navigation may
|
||||
//! fail and is simply retried with backoff.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use chromiumoxide::browser::{Browser, BrowserConfig};
|
||||
use chromiumoxide::cdp::browser_protocol::network::{CookieParam, CookieSameSite};
|
||||
use chromiumoxide::cdp::js_protocol::runtime::{
|
||||
ConsoleApiCalledType, EvaluateParams, EventConsoleApiCalled,
|
||||
};
|
||||
@@ -22,17 +24,7 @@ use serde_json::Value;
|
||||
use silverbullet_server::runtime::{LogBuffer, LogEntry, RuntimeError};
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
|
||||
use crate::config::ChromeConfig;
|
||||
|
||||
/// The live browser and its single page, kept together so dropping the pair
|
||||
/// (on restart) tears down the old Chrome process cleanly.
|
||||
pub struct Live {
|
||||
/// Held purely to keep the Chrome process alive; dropping `Live` (on
|
||||
/// restart) drops this and tears Chrome down. Never read directly.
|
||||
#[allow(dead_code)]
|
||||
pub browser: Browser,
|
||||
pub page: Page,
|
||||
}
|
||||
use crate::config::{ChromeConfig, SpacePage};
|
||||
|
||||
/// Evaluate a raw JS expression in the page with await-promise +
|
||||
/// return-by-value semantics, returning its JSON value (`Null` when the
|
||||
@@ -103,6 +95,18 @@ async fn eval_sync(page: &Page, js: &str) -> Result<Value, RuntimeError> {
|
||||
Ok(result.value().cloned().unwrap_or(Value::Null))
|
||||
}
|
||||
|
||||
/// Bound on the CDP round trips this module makes against a page it has reason
|
||||
/// to distrust: the liveness probe, and every `close`.
|
||||
///
|
||||
/// Nothing here was ever *unbounded* — chromiumoxide gives every command its own
|
||||
/// `REQUEST_TIMEOUT` (~30s) and resolves it to `CdpError::Timeout` — but 30s is
|
||||
/// the wrong bound for these call sites, and inheriting it from a dependency's
|
||||
/// internals leaves the policy invisible. A liveness probe evaluates the literal
|
||||
/// `1` and a close is a single command; five seconds is already enormously
|
||||
/// generous for either, and the difference between 5s and 30s is paid by the
|
||||
/// server (see `page_is_alive`) or by a space's restart latency.
|
||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Current wall-clock time in milliseconds since the Unix epoch.
|
||||
fn now_millis() -> i64 {
|
||||
SystemTime::now()
|
||||
@@ -111,30 +115,20 @@ fn now_millis() -> i64 {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Build the headless page URL from the configured server URL, trimming a
|
||||
/// trailing slash and appending `?headless=1&token=…`. The token is always
|
||||
/// present; an open (no-auth) server simply ignores it.
|
||||
fn page_url(config: &ChromeConfig) -> String {
|
||||
let base = config.server_url.trim_end_matches('/');
|
||||
format!("{base}/?headless=1&token={}", config.headless_token)
|
||||
/// The space's headless auth cookie: HTTP-only, `SameSite=Strict`, scoped by
|
||||
/// `Path` to the space prefix, with no expiry (a session cookie).
|
||||
pub(crate) fn headless_cookie(page: &SpacePage) -> Result<CookieParam, String> {
|
||||
CookieParam::builder()
|
||||
.name(&page.cookie_name)
|
||||
.value(&page.headless_token)
|
||||
.url(&page.server_url)
|
||||
.path(page.cookie_path())
|
||||
.http_only(true)
|
||||
.same_site(CookieSameSite::Strict)
|
||||
.build()
|
||||
}
|
||||
|
||||
/// Launch a fresh browser + page, attach console capture, wait for the client
|
||||
/// runtime to become ready, then publish the live pair into `live_slot` and
|
||||
/// set `ready = true`. Returns `Err(msg)` on any failure (the caller backs off
|
||||
/// and retries).
|
||||
async fn launch_once(
|
||||
config: &ChromeConfig,
|
||||
live_slot: &Arc<Mutex<Option<Live>>>,
|
||||
ready: &Arc<AtomicBool>,
|
||||
logs: &LogBuffer,
|
||||
) -> Result<(), String> {
|
||||
// chromiumoxide's `.args()` expects BARE flag keys (no leading `--`) — it
|
||||
// prepends the dashes itself when building the command line, so a value like
|
||||
// `"--no-sandbox"` is mangled to `----no-sandbox` and silently ignored. Use
|
||||
// the dedicated `.no_sandbox()` method (it emits the correct
|
||||
// `--no-sandbox` + `--disable-setuid-sandbox`, required when running as root,
|
||||
// e.g. inside a container) and pass the remaining flags as bare keys.
|
||||
pub(crate) async fn launch_browser(config: &ChromeConfig) -> Result<Browser, String> {
|
||||
let mut builder = BrowserConfig::builder()
|
||||
.chrome_executable(&config.chrome_path)
|
||||
.user_data_dir(&config.user_data_dir)
|
||||
@@ -164,18 +158,127 @@ async fn launch_once(
|
||||
}
|
||||
});
|
||||
|
||||
let page = browser
|
||||
.new_page(page_url(config).as_str())
|
||||
.await
|
||||
.map_err(|e| format!("new page: {e}"))?;
|
||||
Ok(browser)
|
||||
}
|
||||
|
||||
attach_console_capture(&page, logs, config.log_console)
|
||||
/// A page handle whose tab must be closed explicitly rather than dropped.
|
||||
///
|
||||
/// Abstracted behind a trait for exactly one reason: `PageCloseGuard`'s
|
||||
/// behaviour under task cancellation is the thing this module most needs to
|
||||
/// test, and `chromiumoxide::Page` cannot be constructed without a live Chrome.
|
||||
/// Production only ever instantiates it at `P = Page`.
|
||||
///
|
||||
/// Declared with an explicit `-> impl Future + Send` rather than `async fn`: the
|
||||
/// close is handed to `tokio::spawn`, which needs the `Send` bound stated on the
|
||||
/// trait itself.
|
||||
trait ClosablePage: Send + 'static {
|
||||
fn close_page(self) -> impl std::future::Future<Output = ()> + Send;
|
||||
}
|
||||
|
||||
impl ClosablePage for Page {
|
||||
async fn close_page(self) {
|
||||
// Explicitly bounded, rather than leaning on chromiumoxide's ~30s
|
||||
// request timeout: this frequently runs on a page already declared dead.
|
||||
let _ = tokio::time::timeout(PROBE_TIMEOUT, self.close()).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Owns a freshly opened page and **closes** it on drop, unless disarmed.
|
||||
///
|
||||
/// chromiumoxide implements `Drop` for `Browser` but *not* for `Page` —
|
||||
/// `Page::close` is an explicit `async fn` — so a dropped `Page` leaves its tab
|
||||
/// and renderer process running inside the browser every other space depends
|
||||
/// on. This used to be impossible: each attempt owned its own `Browser`, so a
|
||||
/// failed attempt took the whole process tree with it. Sharing the browser
|
||||
/// removed that implicit cleanup, and this guard replaces it.
|
||||
struct PageCloseGuard<P: ClosablePage> {
|
||||
page: Option<P>,
|
||||
}
|
||||
|
||||
impl<P: ClosablePage> PageCloseGuard<P> {
|
||||
fn new(page: P) -> Self {
|
||||
Self { page: Some(page) }
|
||||
}
|
||||
|
||||
/// The guarded page. Borrowed, never moved out: ownership stays with the
|
||||
/// guard so cancellation at *any* await point still reaches `Drop`.
|
||||
fn page(&self) -> &P {
|
||||
self.page.as_ref().expect("armed until disarmed")
|
||||
}
|
||||
|
||||
/// Give up responsibility for the page. Only correct once the page is
|
||||
/// published somewhere that will close it — i.e. `live`.
|
||||
fn disarm(&mut self) {
|
||||
self.page = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl<P: ClosablePage> Drop for PageCloseGuard<P> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(page) = self.page.take() {
|
||||
// Fire-and-forget: `Drop` cannot await. See the type-level comment
|
||||
// for why a runtime is guaranteed to be present here.
|
||||
tokio::spawn(page.close_page());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Open this space's page in the shared browser: blank page, auth cookie,
|
||||
/// console capture, navigation, then wait for the client runtime to report
|
||||
/// ready. Publishes into `live` and sets `ready` on success.
|
||||
///
|
||||
/// The tab is owned by a `PageCloseGuard` for the whole of this function, so
|
||||
/// every exit — `?`, a returned `Err`, or the task being aborted at any of the
|
||||
/// await points — closes it. That is the single mechanism; there is
|
||||
/// deliberately no separate close on the error paths. Without it, a space that
|
||||
/// keeps timing out in `wait_for_client_ready` (a large space still indexing on
|
||||
/// first boot is the likeliest trigger), or one whose supervisor is aborted
|
||||
/// mid-launch by an admin API space write, abandons a live renderer per attempt,
|
||||
/// forever — the browser never looks dead, so nothing ever retires it.
|
||||
///
|
||||
/// One residual window is out of reach: an abort landing *inside* `new_page`
|
||||
/// itself can leave a target that was created browser-side but whose handle
|
||||
/// never reached this task. It is a single CDP round trip rather than a 60s
|
||||
/// poll loop, so the exposure is tiny compared to what the guard covers.
|
||||
async fn launch_page(
|
||||
browser: &Browser,
|
||||
page_cfg: &SpacePage,
|
||||
log_console: bool,
|
||||
live: &Arc<Mutex<Option<Page>>>,
|
||||
ready: &Arc<AtomicBool>,
|
||||
logs: &LogBuffer,
|
||||
) -> Result<(), String> {
|
||||
let mut guard = PageCloseGuard::new(
|
||||
browser
|
||||
.new_page("about:blank")
|
||||
.await
|
||||
.map_err(|e| format!("new page: {e}"))?,
|
||||
);
|
||||
let page = guard.page();
|
||||
|
||||
page.set_cookie(headless_cookie(page_cfg)?)
|
||||
.await
|
||||
.map_err(|e| format!("headless auth cookie: {e}"))?;
|
||||
|
||||
// Console capture is attached *before* navigating: attaching afterwards
|
||||
// deterministically misses the client's earliest boot output, which is
|
||||
// exactly what an operator needs to debug a space that never reaches ready.
|
||||
attach_console_capture(page, logs, log_console)
|
||||
.await
|
||||
.map_err(|e| format!("console capture: {e}"))?;
|
||||
|
||||
wait_for_client_ready(&page).await?;
|
||||
page.goto(page_cfg.page_url().as_str())
|
||||
.await
|
||||
.map_err(|e| format!("headless page navigation: {e}"))?;
|
||||
|
||||
*live_slot.lock().await = Some(Live { browser, page });
|
||||
wait_for_client_ready(page).await?;
|
||||
|
||||
// Publish first, disarm second, and never await in between: from here on the
|
||||
// supervisor (or `SharedChromeTransport::Drop`) closes the page via `live`.
|
||||
// `Page` is `Clone` and the clones share one underlying tab, so closing any
|
||||
// of them closes it.
|
||||
*live.lock().await = Some(page.clone());
|
||||
guard.disarm();
|
||||
ready.store(true, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
@@ -267,29 +370,30 @@ async fn wait_for_client_ready(page: &Page) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Is the current page still alive? A trivial eval that errors means the page
|
||||
/// (or the whole browser) is gone.
|
||||
async fn page_is_alive(live_slot: &Arc<Mutex<Option<Live>>>) -> bool {
|
||||
let guard = live_slot.lock().await;
|
||||
/// Is this space's page still alive? A trivial eval that errors — or that never
|
||||
/// answers — means the page (or the whole browser) is gone.
|
||||
async fn page_is_alive(live: &Arc<Mutex<Option<Page>>>) -> bool {
|
||||
let guard = live.lock().await;
|
||||
match guard.as_ref() {
|
||||
None => false,
|
||||
Some(live) => eval_sync(&live.page, "1").await.is_ok(),
|
||||
// A timed-out probe reports *not* alive, so the supervisor takes the
|
||||
// restart path rather than waiting on a page that will never answer.
|
||||
Some(page) => matches!(
|
||||
tokio::time::timeout(PROBE_TIMEOUT, eval_sync(page, "1")).await,
|
||||
Ok(Ok(_))
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Supervise the browser for the lifetime of the transport.
|
||||
///
|
||||
/// Loop shape: each iteration first checks liveness. When the page is dead
|
||||
/// (the initial `None` slot, or a crashed page), we mark the runtime not-ready,
|
||||
/// drop any stale `Live`, and attempt `launch_once`. On success we reset the
|
||||
/// backoff to its floor and continue; on failure we sleep for the current
|
||||
/// backoff (2s, doubling up to 120s) before the next attempt. When the page is
|
||||
/// alive we sleep ~2s between liveness checks. The browser is launched LAZILY:
|
||||
/// the supervisor idles until the first runtime request (`requested`) before its
|
||||
/// first `launch_once`, so Chrome never starts while the runtime API is unused.
|
||||
pub async fn supervise(
|
||||
config: ChromeConfig,
|
||||
live_slot: Arc<Mutex<Option<Live>>>,
|
||||
async fn browser_is_dead(browser: &Browser) -> bool {
|
||||
browser.pages().await.is_err()
|
||||
}
|
||||
|
||||
/// Supervise one space's page for the lifetime of its transport.
|
||||
pub(crate) async fn supervise_space(
|
||||
pool: Arc<crate::pool::ChromePool>,
|
||||
page_cfg: SpacePage,
|
||||
live: Arc<Mutex<Option<Page>>>,
|
||||
ready: Arc<AtomicBool>,
|
||||
logs: LogBuffer,
|
||||
trigger: Arc<Notify>,
|
||||
@@ -298,41 +402,65 @@ pub async fn supervise(
|
||||
const BACKOFF_CAP: Duration = Duration::from_secs(120);
|
||||
const LIVENESS_INTERVAL: Duration = Duration::from_secs(2);
|
||||
|
||||
// Lazy launch: park here until the first runtime request, so Chrome is not
|
||||
// started (and emits no CDP noise) while the runtime API is unused. `Notify`
|
||||
// suspends the task with no polling; if the request arrived first,
|
||||
// `notify_one` already stored a permit so this returns immediately.
|
||||
trigger.notified().await;
|
||||
tracing::info!(
|
||||
"runtime API used; launching headless Chrome ({})",
|
||||
config.chrome_path
|
||||
);
|
||||
|
||||
let mut backoff = BACKOFF_FLOOR;
|
||||
let mut has_launched = false;
|
||||
loop {
|
||||
if page_is_alive(&live_slot).await {
|
||||
if page_is_alive(&live).await {
|
||||
tokio::time::sleep(LIVENESS_INTERVAL).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
// The page is dead (or never launched): mark not-ready and drop any
|
||||
// stale browser before relaunching. A dead page *after* a successful
|
||||
// launch is a crash/restart, worth flagging distinctly from first boot.
|
||||
if has_launched {
|
||||
tracing::warn!("headless Chrome page died; restarting");
|
||||
tracing::warn!("headless page for {} died; restarting", page_cfg.server_url);
|
||||
}
|
||||
ready.store(false, Ordering::Relaxed);
|
||||
*live_slot.lock().await = None;
|
||||
let stale = live.lock().await.take();
|
||||
if let Some(old) = stale {
|
||||
let _ = tokio::time::timeout(PROBE_TIMEOUT, old.close()).await;
|
||||
}
|
||||
|
||||
match launch_once(&config, &live_slot, &ready, &logs).await {
|
||||
// `browser` is deliberately scoped to this block: the pool holds the
|
||||
// owning handle, and a lingering clone here would keep a dead Chrome
|
||||
// process alive across a relaunch.
|
||||
let attempt = match pool.ensure_browser().await {
|
||||
Ok((generation, browser)) => {
|
||||
let result = launch_page(
|
||||
&browser,
|
||||
&page_cfg,
|
||||
pool.config().log_console,
|
||||
&live,
|
||||
&ready,
|
||||
&logs,
|
||||
)
|
||||
.await;
|
||||
// Retire the browser only when it is genuinely gone. Discarding
|
||||
// on every failed attempt would let one space's persistent
|
||||
// navigation failure keep killing the browser for everyone.
|
||||
let dead = result.is_err() && browser_is_dead(&browser).await;
|
||||
// Drop the transient clone *before* retiring the pool's handle,
|
||||
// so clearing the slot really does release the process tree.
|
||||
drop(browser);
|
||||
if dead {
|
||||
pool.discard_browser(generation).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
|
||||
match attempt {
|
||||
Ok(()) => {
|
||||
has_launched = true;
|
||||
backoff = BACKOFF_FLOOR;
|
||||
tracing::info!("headless Chrome runtime ready");
|
||||
tracing::info!("headless runtime ready for {}", page_cfg.server_url);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("headless chrome launch failed: {e}; retrying in {backoff:?}");
|
||||
tracing::warn!(
|
||||
"headless page for {} failed to start: {e}; retrying in {backoff:?}",
|
||||
page_cfg.server_url
|
||||
);
|
||||
tokio::time::sleep(backoff).await;
|
||||
backoff = (backoff * 2).min(BACKOFF_CAP);
|
||||
}
|
||||
@@ -342,8 +470,190 @@ pub async fn supervise(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FakePage {
|
||||
closed: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl ClosablePage for FakePage {
|
||||
async fn close_page(self) {
|
||||
self.closed.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
async fn launch_page_shaped(
|
||||
page: FakePage,
|
||||
live: Arc<Mutex<Option<FakePage>>>,
|
||||
ready: Arc<AtomicBool>,
|
||||
entered: Arc<Notify>,
|
||||
hold: Arc<Notify>,
|
||||
) {
|
||||
let mut guard = PageCloseGuard::new(page);
|
||||
let page = guard.page();
|
||||
entered.notify_one();
|
||||
|
||||
// set_cookie / attach_console_capture / goto.
|
||||
for _ in 0..3 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
// wait_for_client_ready.
|
||||
hold.notified().await;
|
||||
|
||||
*live.lock().await = Some(page.clone());
|
||||
guard.disarm();
|
||||
ready.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Wait (briefly) for the detached close task spawned by `Drop` to run.
|
||||
async fn await_closes(closed: &Arc<AtomicUsize>, want: usize) -> usize {
|
||||
for _ in 0..200 {
|
||||
let n = closed.load(Ordering::SeqCst);
|
||||
if n >= want {
|
||||
return n;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
closed.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn aborting_a_launch_mid_flight_closes_the_page() {
|
||||
let closed = Arc::new(AtomicUsize::new(0));
|
||||
let live = Arc::new(Mutex::new(None));
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let entered = Arc::new(Notify::new());
|
||||
let hold = Arc::new(Notify::new());
|
||||
|
||||
let task = tokio::spawn(launch_page_shaped(
|
||||
FakePage {
|
||||
closed: closed.clone(),
|
||||
},
|
||||
live.clone(),
|
||||
ready.clone(),
|
||||
entered.clone(),
|
||||
hold.clone(),
|
||||
));
|
||||
|
||||
// Let it reach the readiness wait, then cancel it there.
|
||||
entered.notified().await;
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert_eq!(closed.load(Ordering::SeqCst), 0, "nothing closed yet");
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
|
||||
assert_eq!(
|
||||
await_closes(&closed, 1).await,
|
||||
1,
|
||||
"a cancelled launch must close its page, not drop the handle"
|
||||
);
|
||||
assert!(live.lock().await.is_none());
|
||||
assert!(!ready.load(Ordering::Relaxed));
|
||||
}
|
||||
|
||||
/// Cancellation before the readiness wait — the earlier await points
|
||||
/// (`set_cookie`, console capture, `goto`) — is covered too.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn aborting_a_launch_during_setup_closes_the_page() {
|
||||
let closed = Arc::new(AtomicUsize::new(0));
|
||||
let live = Arc::new(Mutex::new(None));
|
||||
let entered = Arc::new(Notify::new());
|
||||
let hold = Arc::new(Notify::new());
|
||||
|
||||
let task = tokio::spawn(launch_page_shaped(
|
||||
FakePage {
|
||||
closed: closed.clone(),
|
||||
},
|
||||
live.clone(),
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
entered.clone(),
|
||||
hold,
|
||||
));
|
||||
// Abort as soon as the guard exists — no sleep — so the cancellation
|
||||
// lands in the setup awaits rather than the readiness wait.
|
||||
entered.notified().await;
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
|
||||
assert_eq!(await_closes(&closed, 1).await, 1);
|
||||
assert!(live.lock().await.is_none());
|
||||
}
|
||||
|
||||
/// The other half: a launch that *succeeds* must not close the page it just
|
||||
/// published. Ownership moves to `live`, and the supervisor closes it from
|
||||
/// there on restart.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn a_successful_launch_does_not_close_the_page() {
|
||||
let closed = Arc::new(AtomicUsize::new(0));
|
||||
let live = Arc::new(Mutex::new(None));
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let entered = Arc::new(Notify::new());
|
||||
let hold = Arc::new(Notify::new());
|
||||
|
||||
let task = tokio::spawn(launch_page_shaped(
|
||||
FakePage {
|
||||
closed: closed.clone(),
|
||||
},
|
||||
live.clone(),
|
||||
ready.clone(),
|
||||
entered,
|
||||
hold.clone(),
|
||||
));
|
||||
hold.notify_one();
|
||||
task.await.unwrap();
|
||||
|
||||
// Long enough that a stray close task would have landed by now.
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
assert_eq!(closed.load(Ordering::SeqCst), 0);
|
||||
assert!(live.lock().await.is_some());
|
||||
assert!(ready.load(Ordering::Relaxed));
|
||||
}
|
||||
|
||||
/// An `Err` return closes the tab through the same single mechanism — there
|
||||
/// is deliberately no separate close on `launch_page`'s error paths.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn an_error_return_closes_the_page_via_the_guard() {
|
||||
let closed = Arc::new(AtomicUsize::new(0));
|
||||
let c = closed.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let guard = PageCloseGuard::new(FakePage { closed: c });
|
||||
let _ = guard.page();
|
||||
Err::<(), String>("headless page navigation: boom".to_string())
|
||||
});
|
||||
assert!(task.await.unwrap().is_err());
|
||||
assert_eq!(await_closes(&closed, 1).await, 1);
|
||||
}
|
||||
|
||||
fn page_cfg(server_url: &str) -> SpacePage {
|
||||
SpacePage {
|
||||
server_url: server_url.to_string(),
|
||||
headless_token: "secret".to_string(),
|
||||
cookie_name: "silverbullet_headless_a".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headless_cookie_is_http_only_and_named_per_space() {
|
||||
let cookie = headless_cookie(&page_cfg("http://127.0.0.1:3000")).unwrap();
|
||||
assert_eq!(cookie.name, "silverbullet_headless_a");
|
||||
assert_eq!(cookie.value, "secret");
|
||||
assert_eq!(cookie.url.as_deref(), Some("http://127.0.0.1:3000"));
|
||||
assert_eq!(cookie.path.as_deref(), Some("/"));
|
||||
assert_eq!(cookie.http_only, Some(true));
|
||||
assert_eq!(cookie.same_site, Some(CookieSameSite::Strict));
|
||||
// Session cookie: it must not outlive the browser.
|
||||
assert_eq!(cookie.expires, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headless_cookie_is_scoped_to_the_space_prefix() {
|
||||
let cookie = headless_cookie(&page_cfg("http://127.0.0.1:3000/notes")).unwrap();
|
||||
assert_eq!(cookie.path.as_deref(), Some("/notes"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn console_level_maps_severity() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
//! The headless-Chrome `ClientTransport`: owns a dedicated tokio runtime and a
|
||||
//! supervised browser page running the normal client in `?headless=1` mode.
|
||||
//!
|
||||
//! `eval_js`/`wait_ready` are synchronous and block the calling thread (the
|
||||
//! server invokes them via `spawn_blocking`); they run their async work on the
|
||||
//! transport's owned runtime. Launch is LAZY and tolerant: the supervisor does
|
||||
//! not start Chrome until the first runtime request (`eval_js`/`wait_ready`),
|
||||
//! then brings the browser up and keeps it alive — so the server can finish
|
||||
//! binding its port before the headless page connects, and Chrome never runs
|
||||
//! while the runtime API is unused. Until the page reports `sbRuntime.ready`,
|
||||
//! `is_ready()` is false and eval/wait return `NotReady`/`Timeout` (→ the
|
||||
//! server answers 503/504).
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde_json::Value;
|
||||
use silverbullet_server::runtime::{ClientTransport, LogBuffer, RuntimeError};
|
||||
use tokio::runtime::Runtime;
|
||||
use tokio::sync::{Mutex, Notify};
|
||||
|
||||
use crate::config::ChromeConfig;
|
||||
use crate::supervisor::{eval_on_page, Live};
|
||||
|
||||
pub struct ChromeTransport {
|
||||
rt: Option<Runtime>,
|
||||
live: Arc<Mutex<Option<Live>>>,
|
||||
ready: Arc<AtomicBool>,
|
||||
/// Notified on the first runtime request; the supervisor parks on this
|
||||
/// before launching Chrome, so the browser only starts when the runtime API
|
||||
/// is actually used.
|
||||
trigger: Arc<Notify>,
|
||||
_supervisor: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl ChromeTransport {
|
||||
/// Launch Chrome and start the supervisor. Pushes console output into `logs`
|
||||
/// (pass a clone of the `LogBuffer` you give to `ClientRuntime`). Returns an
|
||||
/// error only if the tokio runtime can't be created — the browser launch and
|
||||
/// page readiness are handled (and retried) by the supervisor, so the server
|
||||
/// can finish binding its port before the headless page connects.
|
||||
pub fn launch(config: ChromeConfig, logs: LogBuffer) -> Result<Self, RuntimeError> {
|
||||
let rt = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(2)
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|e| RuntimeError::Transport(format!("tokio runtime: {e}")))?;
|
||||
let live = Arc::new(Mutex::new(None));
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let trigger = Arc::new(Notify::new());
|
||||
let supervisor = {
|
||||
let (live, ready, logs, config, trigger) = (
|
||||
live.clone(),
|
||||
ready.clone(),
|
||||
logs.clone(),
|
||||
config.clone(),
|
||||
trigger.clone(),
|
||||
);
|
||||
rt.spawn(crate::supervisor::supervise(
|
||||
config, live, ready, logs, trigger,
|
||||
))
|
||||
};
|
||||
Ok(Self {
|
||||
rt: Some(rt),
|
||||
live,
|
||||
ready,
|
||||
trigger,
|
||||
_supervisor: supervisor,
|
||||
})
|
||||
}
|
||||
|
||||
/// The owned runtime. Always present until `Drop`; the `Option` exists only
|
||||
/// so `Drop` can take it.
|
||||
fn rt(&self) -> &Runtime {
|
||||
self.rt.as_ref().expect("runtime present until drop")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ChromeTransport {
|
||||
fn drop(&mut self) {
|
||||
if let Some(rt) = self.rt.take() {
|
||||
rt.shutdown_background();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientTransport for ChromeTransport {
|
||||
fn eval_js(&self, js: &str, timeout: Duration) -> Result<Value, RuntimeError> {
|
||||
// Any runtime use wakes the supervisor to launch Chrome.
|
||||
self.trigger.notify_one();
|
||||
let live = self.live.clone();
|
||||
let js = js.to_string();
|
||||
self.rt().block_on(async move {
|
||||
let guard = live.lock().await;
|
||||
let live = guard.as_ref().ok_or(RuntimeError::NotReady)?;
|
||||
match tokio::time::timeout(timeout, eval_on_page(&live.page, &js)).await {
|
||||
Err(_) => Err(RuntimeError::Timeout),
|
||||
Ok(r) => r,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn wait_ready(&self, timeout: Duration) -> Result<(), RuntimeError> {
|
||||
// First runtime request wakes the supervisor to launch Chrome.
|
||||
self.trigger.notify_one();
|
||||
if self.ready.load(Ordering::Relaxed) {
|
||||
return Ok(());
|
||||
}
|
||||
let ready = self.ready.clone();
|
||||
self.rt().block_on(async move {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
if ready.load(Ordering::Relaxed) {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(RuntimeError::NotReady);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn ensure_started(&self) {
|
||||
// Same lazy-launch nudge as eval/wait: wake the supervisor so a
|
||||
// log read alone is enough to bring Chrome up.
|
||||
self.trigger.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use silverbullet_server::runtime::LogBuffer;
|
||||
|
||||
/// Regression test for the Ctrl-C shutdown panic: the transport owns a
|
||||
/// multi-thread runtime and is dropped while the outer server runtime is
|
||||
/// still active (here, the multi-thread `#[tokio::test]` runtime, matching
|
||||
/// `#[tokio::main]`).
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_transport_inside_async_context_does_not_panic() {
|
||||
let cfg = crate::config::ChromeConfig::resolve(
|
||||
Some("/nonexistent/chrome".into()),
|
||||
None,
|
||||
"http://127.0.0.1:0".into(),
|
||||
String::new(),
|
||||
"/tmp".into(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.expect("config resolves with an explicit chrome path");
|
||||
let transport = ChromeTransport::launch(cfg, LogBuffer::new()).unwrap();
|
||||
drop(transport); // must not panic
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eval_against_live_server_when_configured() {
|
||||
let Ok(url) = std::env::var("SB_TEST_RUNTIME_URL") else {
|
||||
eprintln!("skip: set SB_TEST_RUNTIME_URL to run");
|
||||
return;
|
||||
};
|
||||
let Some(cfg) = crate::config::ChromeConfig::resolve(
|
||||
None,
|
||||
None,
|
||||
url,
|
||||
String::new(),
|
||||
"/tmp/x".into(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
) else {
|
||||
eprintln!("skip: no Chrome");
|
||||
return;
|
||||
};
|
||||
let t = ChromeTransport::launch(cfg, LogBuffer::new()).unwrap();
|
||||
t.wait_ready(std::time::Duration::from_secs(60)).unwrap();
|
||||
let v = t
|
||||
.eval_js(
|
||||
"sbRuntime.evalLua(\"return 1 + 1\")",
|
||||
std::time::Duration::from_secs(10),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v, serde_json::json!(2));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user