chore(conf): add render control

Author: j-mendez

Cargo.lock

@@ -173,3 +173,27 @@ DATE: 2025-02-25 23:34:17
 Total Duration: 6.259639875s
 Average Duration: 125.172952ms
 
+<http://spider.cloud> - 20 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:28:37
+Total Duration: 2.627886125s
+Average Duration: 131.371589ms
+
+<http://spider.cloud> - 20 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:33:23
+Total Duration: 2.608605958s
+Average Duration: 130.414008ms
+
+<http://spider.cloud> - 10 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:34:13
+Total Duration: 1.338665375s
+Average Duration: 133.848741ms
+

benches/logs/Darwin_v10cpu_benchmark_logs.txt

@@ -22,3 +22,28 @@ DATE: 2025-02-25 23:34:09
 Total Duration: 4.158599583s
 Average Duration: 3.755785809s
 
+<http://spider.cloud> - 20 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (99)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle,--no-first-run,--no-sandbox,--disable-setuid-sandbox,--no-zygote,--hide-scrollbars,--user-data-dir=~/.config/google-chrome,--allow-running-insecure-content,--autoplay-policy=user-gesture-required,--ignore-certificate-errors,--no-default-browser-check,--disable-dev-shm-usage,--disable-threaded-scrolling,--disable-cookie-encryption,--disable-demo-mode,--disable-dinosaur-easter-egg,--disable-fetching-hints-at-navigation-start,--disable-site-isolation-trials,--disable-web-security,--disable-threaded-animation,--disable-sync,--disable-print-preview,--disable-search-engine-choice-screen,--disable-partial-raster,--disable-in-process-stack-traces,--use-angle=swiftshader,--disable-low-res-tiling,--disable-speech-api,--disable-oobe-chromevox-hint-timer-for-testing,--disable-smooth-scrolling,--disable-default-apps,--disable-prompt-on-repost,--disable-domain-reliability,--enable-dom-distiller,--enable-distillability-service,--disable-component-update,--disable-background-timer-throttling,--disable-breakpad,--disable-crash-reporter,--disable-software-rasterizer,--disable-asynchronous-spellchecking,--disable-extensions,--disable-html5-camera,--noerrdialogs,--disable-popup-blocking,--disable-hang-monitor,--disable-checker-imaging,--enable-surface-synchronization,--disable-image-animation-resync,--disable-client-side-phishing-detection,--disable-component-extensions-with-background-pages,--run-all-compositor-stages-before-draw,--disable-background-networking,--disable-renderer-backgrounding,--disable-field-trial-config,--disable-back-forward-cache,--disable-backgrounding-occluded-windows,--log-level=3,--enable-logging=stderr,--font-render-hinting=none,--block-new-web-contents,--no-subproc-heap-profiling,--no-pre-read-main-dll,--disable-stack-profiler,--disable-libassistant-logfile,--crash-on-hang-threads,--restore-last-session,--ip-protection-proxy-opt-out,--unsafely-disable-devtools-self-xss-warning,--enable-features=PdfOopif,SharedArrayBuffer,NetworkService,NetworkServiceInProcess,--metrics-recording-only,--use-mock-keychain,--force-color-profile=srgb,--disable-infobars,--mute-audio,--disable-datasaver-prompt,--no-service-autorun,--password-store=basic,--export-tagged-pdf,--no-pings,--rusty-png,--disable-histogram-customizer,--window-size=800,600,--disable-vulkan-fallback-to-gl-for-testing,--disable-vulkan-surface,--disable-webrtc,--disable-oopr-debug-crash-dump,--disable-pnacl-crash-throttling,--disable-renderer-accessibility,--renderer-process-limit=50,--disable-pushstate-throttle,--disable-blink-features=AutomationControlled,--disable-ipc-flooding-protection,--disable-features=PaintHolding,HttpsUpgrades,DeferRendererTasksAfterInput,LensOverlay,ThirdPartyStoragePartitioning,IsolateSandboxedIframes,ProcessPerSiteUpToMainFrameThreshold,site-per-process,WebUIJSErrorReportingExtended,DIPS,InterestFeedContentSuggestions,PrivacySandboxSettings4,AutofillServerCommunication,CalculateNativeWinOcclusion,OptimizationHints,AudioServiceOutOfProcess,IsolateOrigins,ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:28:33
+Total Duration: 1.189235583s
+Average Duration: 753.532166ms
+
+<http://spider.cloud> - 20 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (99)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle,--no-first-run,--no-sandbox,--disable-setuid-sandbox,--no-zygote,--hide-scrollbars,--user-data-dir=~/.config/google-chrome,--allow-running-insecure-content,--autoplay-policy=user-gesture-required,--ignore-certificate-errors,--no-default-browser-check,--disable-dev-shm-usage,--disable-threaded-scrolling,--disable-cookie-encryption,--disable-demo-mode,--disable-dinosaur-easter-egg,--disable-fetching-hints-at-navigation-start,--disable-site-isolation-trials,--disable-web-security,--disable-threaded-animation,--disable-sync,--disable-print-preview,--disable-search-engine-choice-screen,--disable-partial-raster,--disable-in-process-stack-traces,--use-angle=swiftshader,--disable-low-res-tiling,--disable-speech-api,--disable-oobe-chromevox-hint-timer-for-testing,--disable-smooth-scrolling,--disable-default-apps,--disable-prompt-on-repost,--disable-domain-reliability,--enable-dom-distiller,--enable-distillability-service,--disable-component-update,--disable-background-timer-throttling,--disable-breakpad,--disable-crash-reporter,--disable-software-rasterizer,--disable-asynchronous-spellchecking,--disable-extensions,--disable-html5-camera,--noerrdialogs,--disable-popup-blocking,--disable-hang-monitor,--disable-checker-imaging,--enable-surface-synchronization,--disable-image-animation-resync,--disable-client-side-phishing-detection,--disable-component-extensions-with-background-pages,--run-all-compositor-stages-before-draw,--disable-background-networking,--disable-renderer-backgrounding,--disable-field-trial-config,--disable-back-forward-cache,--disable-backgrounding-occluded-windows,--log-level=3,--enable-logging=stderr,--font-render-hinting=none,--block-new-web-contents,--no-subproc-heap-profiling,--no-pre-read-main-dll,--disable-stack-profiler,--disable-libassistant-logfile,--crash-on-hang-threads,--restore-last-session,--ip-protection-proxy-opt-out,--unsafely-disable-devtools-self-xss-warning,--enable-features=PdfOopif,SharedArrayBuffer,NetworkService,NetworkServiceInProcess,--metrics-recording-only,--use-mock-keychain,--force-color-profile=srgb,--disable-infobars,--mute-audio,--disable-datasaver-prompt,--no-service-autorun,--password-store=basic,--export-tagged-pdf,--no-pings,--rusty-png,--disable-histogram-customizer,--window-size=800,600,--disable-vulkan-fallback-to-gl-for-testing,--disable-vulkan-surface,--disable-webrtc,--disable-oopr-debug-crash-dump,--disable-pnacl-crash-throttling,--disable-renderer-accessibility,--renderer-process-limit=50,--disable-pushstate-throttle,--disable-blink-features=AutomationControlled,--disable-ipc-flooding-protection,--disable-features=PaintHolding,HttpsUpgrades,DeferRendererTasksAfterInput,LensOverlay,ThirdPartyStoragePartitioning,IsolateSandboxedIframes,ProcessPerSiteUpToMainFrameThreshold,site-per-process,WebUIJSErrorReportingExtended,DIPS,InterestFeedContentSuggestions,PrivacySandboxSettings4,AutofillServerCommunication,CalculateNativeWinOcclusion,OptimizationHints,AudioServiceOutOfProcess,IsolateOrigins,ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:33:19
+Total Duration: 1.187148708s
+Average Duration: 751.300475ms
+
+<http://spider.cloud> - 10 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (99)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle,--no-first-run,--no-sandbox,--disable-setuid-sandbox,--no-zygote,--hide-scrollbars,--user-data-dir=~/.config/google-chrome,--allow-running-insecure-content,--autoplay-policy=user-gesture-required,--ignore-certificate-errors,--no-default-browser-check,--disable-dev-shm-usage,--disable-threaded-scrolling,--disable-cookie-encryption,--disable-demo-mode,--disable-dinosaur-easter-egg,--disable-fetching-hints-at-navigation-start,--disable-site-isolation-trials,--disable-web-security,--disable-threaded-animation,--disable-sync,--disable-print-preview,--disable-search-engine-choice-screen,--disable-partial-raster,--disable-in-process-stack-traces,--use-angle=swiftshader,--disable-low-res-tiling,--disable-speech-api,--disable-oobe-chromevox-hint-timer-for-testing,--disable-smooth-scrolling,--disable-default-apps,--disable-prompt-on-repost,--disable-domain-reliability,--enable-dom-distiller,--enable-distillability-service,--disable-component-update,--disable-background-timer-throttling,--disable-breakpad,--disable-crash-reporter,--disable-software-rasterizer,--disable-asynchronous-spellchecking,--disable-extensions,--disable-html5-camera,--noerrdialogs,--disable-popup-blocking,--disable-hang-monitor,--disable-checker-imaging,--enable-surface-synchronization,--disable-image-animation-resync,--disable-client-side-phishing-detection,--disable-component-extensions-with-background-pages,--run-all-compositor-stages-before-draw,--disable-background-networking,--disable-renderer-backgrounding,--disable-field-trial-config,--disable-back-forward-cache,--disable-backgrounding-occluded-windows,--log-level=3,--enable-logging=stderr,--font-render-hinting=none,--block-new-web-contents,--no-subproc-heap-profiling,--no-pre-read-main-dll,--disable-stack-profiler,--disable-libassistant-logfile,--crash-on-hang-threads,--restore-last-session,--ip-protection-proxy-opt-out,--unsafely-disable-devtools-self-xss-warning,--enable-features=PdfOopif,SharedArrayBuffer,NetworkService,NetworkServiceInProcess,--metrics-recording-only,--use-mock-keychain,--force-color-profile=srgb,--disable-infobars,--mute-audio,--disable-datasaver-prompt,--no-service-autorun,--password-store=basic,--export-tagged-pdf,--no-pings,--rusty-png,--disable-histogram-customizer,--window-size=800,600,--disable-vulkan-fallback-to-gl-for-testing,--disable-vulkan-surface,--disable-webrtc,--disable-oopr-debug-crash-dump,--disable-pnacl-crash-throttling,--disable-renderer-accessibility,--renderer-process-limit=50,--disable-pushstate-throttle,--disable-blink-features=AutomationControlled,--disable-ipc-flooding-protection,--disable-features=PaintHolding,HttpsUpgrades,DeferRendererTasksAfterInput,LensOverlay,ThirdPartyStoragePartitioning,IsolateSandboxedIframes,ProcessPerSiteUpToMainFrameThreshold,site-per-process,WebUIJSErrorReportingExtended,DIPS,InterestFeedContentSuggestions,PrivacySandboxSettings4,AutofillServerCommunication,CalculateNativeWinOcclusion,OptimizationHints,AudioServiceOutOfProcess,IsolateOrigins,ImprovedCookieControls,LazyFrameLoading,GlobalMediaControls,DestroyProfileOnBrowserClose,MediaRouter,DialMediaRouteProvider,AcceptCHFrame,AutoExpandDetailsElement,CertificateTransparencyComponentUpdater,AvoidUnnecessaryBeforeUnloadCheckSync,Translate")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:34:10
+Total Duration: 558.842458ms
+Average Duration: 356.873224ms
+
+

benches/logs/Darwin_v10cpu_benchmark_noargs_logs.txt

@@ -22,3 +22,29 @@ DATE: 2025-02-25 23:34:21
 Total Duration: 4.252429458s
 Average Duration: 3.912185341s
 
+<http://spider.cloud> - 20 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:28:38
+Total Duration: 1.195677s
+Average Duration: 761.019604ms
+
+<http://spider.cloud> - 20 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:33:24
+Total Duration: 1.208197125s
+Average Duration: 760.903531ms
+
+<http://spider.cloud> - 10 SAMPLES
+CHROME_PATH: headless_shell
+CHROME_ARGS: (6)("--remote-debugging-address=0.0.0.0,--remote-debugging-port=9223,--headless,--disable-gpu,--disable-gpu-sandbox,--use-gl=angle")
+MACHINE: Darwin/v10cpu
+DATE: 2025-02-26 05:34:13
+Total Duration: 576.73775ms
+Average Duration: 368.845141ms
+
+
+

benches/logs_concurrent/Darwin_v10cpu_benchmark_logs.txt

@@ -23,6 +23,8 @@ openssl-sys = { version = "0.9", features = ["vendored"] }
 cached = { version = "0", features = ["async_tokio_rt_multi_thread"]}
 tracing = "0.1"
 rand = "0.9"
+num_cpus = "1"
+sysinfo = "0.33"
 
 [features]
 testing = []

benches/logs_concurrent/Darwin_v10cpu_benchmark_noargs_logs.txt

@@ -2,7 +2,7 @@ use std::collections::HashSet;
 use std::sync::atomic::{AtomicBool, AtomicU64};
 
 /// The performance arg count.
-pub(crate) const PERF_ARGS: usize = 97;
+pub(crate) const PERF_ARGS: usize = 99;
 
 #[cfg(any(test, feature = "testing"))]
 lazy_static::lazy_static! {
@@ -253,6 +253,8 @@ lazy_static::lazy_static! {
             "--disable-oopr-debug-crash-dump",
             "--disable-pnacl-crash-throttling",
             "--disable-renderer-accessibility",
+            &crate::render_conf::RENDER_PROCESS_LIMIT,
+            "--disable-pushstate-throttle",
             "--disable-blink-features=AutomationControlled",
             "--disable-ipc-flooding-protection", // we do not need to throttle navigation for https://github.com/spider-rs/spider/commit/9ff5bbd7a2656b8edb84b62843b72ae9d09af079#diff-75ce697faf0d37c3dff4a3a19e7524798b3cb5487f8f54beb5d04c4d48e34234R446.
             // --deterministic-mode 10-20% drop in perf

headless_browser_lib/Cargo.toml

@@ -6,6 +6,8 @@ pub mod conf;
 mod modify;
 /// Proxy forwarder TCP to chrome instances.
 pub mod proxy;
+/// Chrome renderer configuration.
+mod render_conf;
 
 use conf::{
     CACHEABLE, CHROME_ADDRESS, CHROME_ARGS, CHROME_INSTANCES, CHROME_PATH, DEBUG_JSON,