chore(docs): prep auto scale load balancer

Author: j-mendez

Cargo.lock

@@ -1,31 +1,34 @@
 # headless-browser
 
-Headless Browser with Proxy and Server.
+The `headless-browser` crate offers a scalable solution for managing headless Chrome instances with integrated proxy and server support. This system is designed to optimize resource utilization and enhance performance for large-scale web automation tasks.
 
 ## Installation
 
-Make sure to have [Rust](https://www.rust-lang.org/learn/get-started) installed.
+Make sure you have [Rust](https://www.rust-lang.org/learn/get-started) installed before proceeding.
 
-`cargo install headless_browser`
+```sh
+cargo install headless_browser
+```
 
 ## Usage
 
-1. Runs the latest browser instance with remote proxy connections and a server that can rewrite json/version for networking.
-1. Can spawn and shutdown multiple headless instances manually and automatically on errors.
-1. Get CDP ws connections and status (caches values for perf).
+1. Launches the most recent browser instance, supporting remote proxy connections and a server for `json/version` rewrites to facilitate networking.
+2. Allows manual and automatic spawning and termination of multiple headless instances, including error handling functionalities.
+3. Provides access to Chrome DevTools Protocol (CDP) WebSocket connections and status, caching values for improved performance.
 
-The current instance binds chrome to 0.0.0.0 when starting via API.
+By default, the instance binds Chrome to `0.0.0.0` when initialized via the API.
 
-Use the env variable `REMOTE_ADDRESS` to change the address of the chrome instance between physical or network.
+Use the `REMOTE_ADDRESS` environment variable to specify the desired address for the Chrome instance, whether local or networked.
 
-The application will pass lb health checks when using port `6000` to get the status of the chromium container.
+The application passes load balancer health checks on port `6000`, providing the status of the Chrome container.
 
-A side loaded application is required to run chromium on a load balancer, one of the main purposes of the server.
+To run Chrome on a load balancer, a companion application is required, which is a primary function of the server.
 
-The default port is `9223` for chromium and `9222` for the TCP proxy to connect to the instance due to `0.0.0.0` not being exposed on latest `HeadlessChrome/131.0.6778.139` and up.
+The default port configuration includes `9223` for Chrome and `9222` for the TCP proxy, in response to `0.0.0.0` not being exposed in recent versions like `HeadlessChrome/131.0.6778.139` and newer.
 
-It is recommended to use the `headless_shell_playwright` docker build or [chrome-headless-shell](https://storage.googleapis.com/chrome-for-testing-public/134.0.6998.23/linux64/chrome-headless-shell-linux64.zip) for web scraping (headless-shell is x10 times faster than headless = new).
+For web scraping, using the `headless_shell_playwright` Docker build or downloading [chrome-headless-shell](https://storage.googleapis.com/chrome-for-testing-public/134.0.6998.23/linux64/chrome-headless-shell-linux64.zip) is recommended, as headless-shell is significantly faster than traditional headless mode.
 
+---
 ## API
 
 1. POST: `fork` to start a new chrome instance or use `fork/$port` with the port to startup the instance ex: `curl --location --request POST 'http://localhost:6000/fork/9223'`.

README.md

@@ -1,5 +1,6 @@
 mod run;
 mod run_concurrent;
+mod utils;
 
 use std::env::set_var;
 
@@ -14,7 +15,7 @@ async fn main() {
         .parse::<u32>()
         .unwrap_or(10);
 
-    headless_browser_lib::fork(Some(*headless_browser_lib::conf::DEFAULT_PORT)).await;
+    headless_browser_lib::fork(Some(*headless_browser_lib::conf::DEFAULT_PORT));
     let task = tokio::spawn(headless_browser_lib::run_main());
     tokio::time::sleep(std::time::Duration::from_millis(1000)).await; // Wait for the server to load.
     run::run(LOG_FILE_NAME, samples).await;

benches/basic.rs

@@ -1,5 +1,7 @@
 mod run;
 mod run_concurrent;
+mod utils;
+
 use std::env::set_var;
 
 const LOG_FILE_NAME: &str = "benchmark_noargs_logs.txt";
@@ -14,7 +16,7 @@ async fn main() {
         .parse::<u32>()
         .unwrap_or(10);
 
-    headless_browser_lib::fork(Some(*headless_browser_lib::conf::DEFAULT_PORT)).await;
+    headless_browser_lib::fork(Some(*headless_browser_lib::conf::DEFAULT_PORT));
     let task = tokio::spawn(headless_browser_lib::run_main());
     tokio::time::sleep(std::time::Duration::from_millis(1000)).await; // Wait for the server to load.
     run::run(LOG_FILE_NAME, samples).await;

benches/basic_no_args.rs

@@ -51,4 +51,13 @@ CHROME_ARGS: (99)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=92
 MACHINE: Darwin/v10cpu
 DATE: 2025-02-26 05:34:10
 Total Duration: 1.328313334s
-Average Duration: 132.799387ms
+Average Duration: 132.799387ms
+
+<http://spider.cloud> - 10 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (99)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle,--no-first-run,--no-sandbox,--disable-setuid-sandbox,--no-zygote,--hide-scrollbars,--user-data-dir=~/.config/google-chrome,--allow-running-insecure-content,--autoplay-policy=user-gesture-required,--ignore-certificate-errors,--no-default-browser-check,--disable-dev-shm-usage,--disable-threaded-scrolling,--disable-cookie-encryption,--disable-demo-mode,--disable-dinosaur-easter-egg,--disable-fetching-hints-at-navigation-start,--disable-site-isolation-trials,--disable-web-security,--disable-threaded-animation,--disable-sync,--disable-print-preview,--disable-search-engine-choice-screen,--disable-partial-raster,--disable-in-process-stack-traces,--use-angle=swiftshader,--disable-low-res-tiling,--disable-speech-api,--disable-oobe-chromevox-hint-timer-for-testing,--disable-smooth-scrolling,--disable-default-apps,--disable-prompt-on-repost,--disable-domain-reliability,--enable-dom-distiller,--enable-distillability-service,--disable-component-update,--disable-background-timer-throttling,--disable-breakpad,--disable-crash-reporter,--disable-software-rasterizer,--disable-asynchronous-spellchecking,--disable-extensions,--disable-html5-camera,--noerrdialogs,--disable-popup-blocking,--disable-hang-monitor,--disable-checker-imaging,--enable-surface-synchronization,--disable-image-animation-resync,--disable-client-side-phishing-detection,--disable-component-extensions-with-background-pages,--run-all-compositor-stages-before-draw,--disable-background-networking,--disable-renderer-backgrounding,--disable-field-trial-config,--disable-back-forward-cache,--disable-backgrounding-occluded-windows,--log-level=3,--enable-logging=stderr,--font-render-hinting=none,--block-new-web-contents,--no-subproc-heap-profiling,--no-pre-read-main-dll,--disable-stack-profiler,--disable-libassistant-logfile,--crash-on-hang-threads,--restore-last-session,--ip-protection-proxy-opt-out,--unsafely-disable-devtools-self-xss-warning,--enable-features=PdfOopif,SharedArrayBuffer,NetworkService,NetworkServiceInProcess,--metrics-recording-only,--use-mock-keychain,--force-color-profile=srgb,--disable-infobars,--mute-audio,--disable-datasaver-prompt,--no-service-autorun,--password-store=basic,--export-tagged-pdf,--no-pings,--rusty-png,--disable-histogram-customizer,--window-size=800,600,--disable-vulkan-fallback-to-gl-for-testing,--disable-vulkan-surface,--disable-webrtc,--disable-oopr-debug-crash-dump,--disable-pnacl-crash-throttling,--disable-renderer-accessibility,--renderer-process-limit=50,--disable-pushstate-throttle,--disable-blink-features=AutomationControlled,--disable-ipc-flooding-protection,--disable-features=PaintHolding,HttpsUpgrades,DeferRendererTasksAfterInput,LensOverlay,ThirdPartyStoragePartitioning,IsolateSandboxedIframes,ProcessPerSiteUpToMainFrameThreshold,site-per-process,WebUIJSErrorReportingExtended,DIPS,InterestFeedContentSuggestions,PrivacySandboxSettings4,AutofillServerCommunication,CalculateNativeWinOcclusion,OptimizationHints,AudioServiceOutOfProcess,IsolateOrigins,ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-27 21:54:40
+Total Duration: 1.451455792s
+Average Duration: 145.12042ms
+

benches/logs/Darwin_v10cpu_benchmark_logs.txt

@@ -197,3 +197,11 @@ DATE: 2025-02-26 05:34:13
 Total Duration: 1.338665375s
 Average Duration: 133.848741ms
 
+<http://spider.cloud> - 10 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-27 21:54:43
+Total Duration: 1.592259375s
+Average Duration: 159.205304ms
+

benches/logs/Darwin_v10cpu_benchmark_noargs_logs.txt

@@ -46,4 +46,11 @@ DATE: 2025-02-26 05:34:10
 Total Duration: 558.842458ms
 Average Duration: 356.873224ms
 
+<http://spider.cloud> - 10 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (99)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle,--no-first-run,--no-sandbox,--disable-setuid-sandbox,--no-zygote,--hide-scrollbars,--user-data-dir=~/.config/google-chrome,--allow-running-insecure-content,--autoplay-policy=user-gesture-required,--ignore-certificate-errors,--no-default-browser-check,--disable-dev-shm-usage,--disable-threaded-scrolling,--disable-cookie-encryption,--disable-demo-mode,--disable-dinosaur-easter-egg,--disable-fetching-hints-at-navigation-start,--disable-site-isolation-trials,--disable-web-security,--disable-threaded-animation,--disable-sync,--disable-print-preview,--disable-search-engine-choice-screen,--disable-partial-raster,--disable-in-process-stack-traces,--use-angle=swiftshader,--disable-low-res-tiling,--disable-speech-api,--disable-oobe-chromevox-hint-timer-for-testing,--disable-smooth-scrolling,--disable-default-apps,--disable-prompt-on-repost,--disable-domain-reliability,--enable-dom-distiller,--enable-distillability-service,--disable-component-update,--disable-background-timer-throttling,--disable-breakpad,--disable-crash-reporter,--disable-software-rasterizer,--disable-asynchronous-spellchecking,--disable-extensions,--disable-html5-camera,--noerrdialogs,--disable-popup-blocking,--disable-hang-monitor,--disable-checker-imaging,--enable-surface-synchronization,--disable-image-animation-resync,--disable-client-side-phishing-detection,--disable-component-extensions-with-background-pages,--run-all-compositor-stages-before-draw,--disable-background-networking,--disable-renderer-backgrounding,--disable-field-trial-config,--disable-back-forward-cache,--disable-backgrounding-occluded-windows,--log-level=3,--enable-logging=stderr,--font-render-hinting=none,--block-new-web-contents,--no-subproc-heap-profiling,--no-pre-read-main-dll,--disable-stack-profiler,--disable-libassistant-logfile,--crash-on-hang-threads,--restore-last-session,--ip-protection-proxy-opt-out,--unsafely-disable-devtools-self-xss-warning,--enable-features=PdfOopif,SharedArrayBuffer,NetworkService,NetworkServiceInProcess,--metrics-recording-only,--use-mock-keychain,--force-color-profile=srgb,--disable-infobars,--mute-audio,--disable-datasaver-prompt,--no-service-autorun,--password-store=basic,--export-tagged-pdf,--no-pings,--rusty-png,--disable-histogram-customizer,--window-size=800,600,--disable-vulkan-fallback-to-gl-for-testing,--disable-vulkan-surface,--disable-webrtc,--disable-oopr-debug-crash-dump,--disable-pnacl-crash-throttling,--disable-renderer-accessibility,--renderer-process-limit=50,--disable-pushstate-throttle,--disable-blink-features=AutomationControlled,--disable-ipc-flooding-protection,--disable-features=PaintHolding,HttpsUpgrades,DeferRendererTasksAfterInput,LensOverlay,ThirdPartyStoragePartitioning,IsolateSandboxedIframes,ProcessPerSiteUpToMainFrameThreshold,site-per-process,WebUIJSErrorReportingExtended,DIPS,InterestFeedContentSuggestions,PrivacySandboxSettings4,AutofillServerCommunication,CalculateNativeWinOcclusion,OptimizationHints,AudioServiceOutOfProcess,IsolateOrigins,ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-27 21:54:41
+Total Duration: 643.615208ms
+Average Duration: 409.253508ms
 

benches/logs_concurrent/Darwin_v10cpu_benchmark_logs.txt

@@ -46,5 +46,11 @@ DATE: 2025-02-26 05:34:13
 Total Duration: 576.73775ms
 Average Duration: 368.845141ms
 
-
+<http://spider.cloud> - 10 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-27 21:54:44
+Total Duration: 596.303208ms
+Average Duration: 361.702ms
 

benches/logs_concurrent/Darwin_v10cpu_benchmark_noargs_logs.txt

@@ -1,10 +1,9 @@
-use chromiumoxide::browser::Browser;
-use futures_util::stream::StreamExt;
+use super::utils::{ensure_log_directory_exists, get_last_benchmark, navigate_extract_and_close};
 use std::ops::Div;
 use std::{
     env,
-    fs::{self, File, OpenOptions},
-    io::{self, BufRead, Write},
+    fs::OpenOptions,
+    io::{self, Write},
     path::Path,
     time::{Duration, Instant},
 };
@@ -46,14 +45,6 @@ pub async fn run(log_file_name: &str, samples: u32) {
         .expect("Failed to log performance");
 }
 
-/// Ensure the dir always exist.
-fn ensure_log_directory_exists(dir: &str) -> io::Result<()> {
-    if !Path::new(dir).exists() {
-        fs::create_dir_all(dir)?;
-    }
-    Ok(())
-}
-
 /// Log the performance to file.
 fn log_performance(
     total_duration: Duration,
@@ -126,57 +117,3 @@ fn log_performance(
     }
     Ok(())
 }
-
-/// Get the last benchmark results duration.
-fn get_last_benchmark(log_file: &File) -> io::Result<Option<Duration>> {
-    let mut lines = io::BufReader::new(log_file).lines();
-    let mut last_line = None;
-    while let Some(line) = lines.next() {
-        last_line = Some(line?);
-    }
-
-    if let Some(last_line) = last_line {
-        if let Some(duration_str) = last_line.split(',').next() {
-            if let Some(duration_value) = duration_str.split(':').nth(1) {
-                return Ok(Some(parse_duration(duration_value.trim())?));
-            }
-        }
-    }
-    Ok(None)
-}
-
-/// Parse the duration without the ms.
-fn parse_duration(s: &str) -> io::Result<Duration> {
-    if let Some(stripped) = s.strip_suffix("ms") {
-        stripped
-            .parse::<f64>()
-            .map(|millis| Duration::from_millis(millis as u64))
-            .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid duration format"))
-    } else {
-        Err(io::Error::new(
-            io::ErrorKind::InvalidData,
-            "Invalid duration format",
-        ))
-    }
-}
-
-/// Navigate, get the HTML, and close the page.
-async fn navigate_extract_and_close(u: &str) -> Result<(), Box<dyn std::error::Error>> {
-    let (browser, mut handler) =
-        Browser::connect_with_config("http://127.0.0.1:6000/json/version", Default::default())
-            .await?;
-
-    let handle = tokio::task::spawn(async move {
-        while let Some(h) = handler.next().await {
-            if h.is_err() {
-                break;
-            }
-        }
-    });
-
-    let page = browser.new_page(u).await?;
-    page.wait_for_navigation().await?.content().await?;
-    handle.abort(); // Abort the handle to drop the connection.
-
-    Ok(())
-}

benches/run.rs

@@ -1,11 +1,10 @@
-use chromiumoxide::browser::Browser;
-use futures_util::stream::StreamExt;
+use super::utils::{ensure_log_directory_exists, get_last_benchmark, navigate_extract_and_close};
 use std::ops::Div;
 use std::sync::Arc;
 use std::{
     env,
-    fs::{self, File, OpenOptions},
-    io::{self, BufRead, Write},
+    fs::OpenOptions,
+    io::{self, Write},
     path::Path,
     time::{Duration, Instant},
 };
@@ -56,14 +55,6 @@ pub async fn run(log_file_name: &str, samples: u32) {
         .expect("Failed to log performance");
 }
 
-/// Ensure the dir always exist.
-fn ensure_log_directory_exists(dir: &str) -> io::Result<()> {
-    if !Path::new(dir).exists() {
-        fs::create_dir_all(dir)?;
-    }
-    Ok(())
-}
-
 /// Log the performance to file.
 fn log_performance(
     total_duration: Duration,
@@ -136,57 +127,3 @@ fn log_performance(
     }
     Ok(())
 }
-
-/// Get the last benchmark results duration.
-fn get_last_benchmark(log_file: &File) -> io::Result<Option<Duration>> {
-    let mut lines = io::BufReader::new(log_file).lines();
-    let mut last_line = None;
-    while let Some(line) = lines.next() {
-        last_line = Some(line?);
-    }
-
-    if let Some(last_line) = last_line {
-        if let Some(duration_str) = last_line.split(',').next() {
-            if let Some(duration_value) = duration_str.split(':').nth(1) {
-                return Ok(Some(parse_duration(duration_value.trim())?));
-            }
-        }
-    }
-    Ok(None)
-}
-
-/// Parse the duration without the ms.
-fn parse_duration(s: &str) -> io::Result<Duration> {
-    if let Some(stripped) = s.strip_suffix("ms") {
-        stripped
-            .parse::<f64>()
-            .map(|millis| Duration::from_millis(millis as u64))
-            .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Invalid duration format"))
-    } else {
-        Err(io::Error::new(
-            io::ErrorKind::InvalidData,
-            "Invalid duration format",
-        ))
-    }
-}
-
-/// Navigate, get the HTML, and close the page.
-async fn navigate_extract_and_close(u: &str) -> Result<(), Box<dyn std::error::Error>> {
-    let (browser, mut handler) =
-        Browser::connect_with_config("http://127.0.0.1:6000/json/version", Default::default())
-            .await?;
-
-    let handle = tokio::task::spawn(async move {
-        while let Some(h) = handler.next().await {
-            if h.is_err() {
-                break;
-            }
-        }
-    });
-
-    let page = browser.new_page(u).await?;
-    page.wait_for_navigation().await?.content().await?;
-    handle.abort(); // Abort the handle to drop the connection.
-
-    Ok(())
-}