Implement HTTP client configuration groups with reusable settings for transport, connection pool, TLS, protocol, and proxy options

- Added `config_groups.rs` to define grouped configuration options for the HTTP client.
- Introduced `TransportConfigOptions`, `PoolConfigOptions`, `TlsConfigOptions`, `ProtocolConfigOptions`, and `ProxyConfigOptions` structs with default values and builder methods for customization.
- Enhanced the `get_all` method in the cookie module to iterate over stored cookies more efficiently.
- Updated hooks service to improve error handling and response processing in the client layer.
- Added assertions in the delay service tests to ensure proper functionality.

Author: Akagi201

Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/*", "bin/*"]
 resolver = "3"
 
 [workspace.package]
-version = "2.4.4"
+version = "2.4.5"
 edition = "2024"
 authors = ["Akagi201 <akagi201@gmail.com>"]
 license = "Apache-2.0"
@@ -185,10 +185,10 @@ all = "warn"
 
 [workspace.dependencies]
 # local crates
-fastwebsockets = { path = "crates/fastwebsockets", package = "hpx-fastwebsockets", version = "2.4.4" }
-hpx = { path = "crates/hpx", version = "2.4.4" }
-hpx-dl = { path = "crates/hpx-dl", version = "2.4.4" }
-hpx-yawc = { path = "crates/yawc", version = "2.4.4" }
+fastwebsockets = { path = "crates/fastwebsockets", package = "hpx-fastwebsockets", version = "2.4.5" }
+hpx = { path = "crates/hpx", version = "2.4.5" }
+hpx-dl = { path = "crates/hpx-dl", version = "2.4.5" }
+hpx-yawc = { path = "crates/yawc", version = "2.4.5" }
 
 # external crates
 ahash = "0.8.12"
@@ -210,7 +210,7 @@ cucumber = "0.22.1"
 digest = "=0.10.7"
 encoding_rs = "0.8.35"
 eyre = "0.6.12"
-fastrand = "2.3.0"
+fastrand = "2.4.1"
 flate2 = "1.1.9"
 futures = "0.3.32"
 futures-channel = "0.3.32"
@@ -220,7 +220,7 @@ futures-util = "0.3.32"
 getrandom = "0.4.2"
 hex = "0.4.3"
 hickory-resolver = "0.25.2"
-hmac = "0.12.1"
+hmac = "0.13.0"
 home = "0.5.12"
 http = "1.4.0"
 http-body = "1.0.1"
@@ -233,7 +233,7 @@ hyper-util = "0.1.20"
 ipnet = "2.12.0"
 js-sys = "0.3.91"
 libc = "0.2.183"
-md-5 = "0.10.6"
+md-5 = "0.10.0"
 memchr = "2.8.0"
 mime = "0.3.17"
 mime_guess = "2.0.5"
@@ -253,14 +253,14 @@ rustls = "0.23.37"
 rustls-native-certs = "0.8.3"
 rustls-pemfile = "2.2.0"
 rustls-pki-types = "1.14.0"
-rustyline = "17.0.2"
+rustyline = "18.0.0"
 scc = "3.6.11"
 schnellru = "0.2.4"
 serde = "1.0.228"
 serde_json = "1.0.149"
 serde_urlencoded = "0.7.1"
-sha1 = "0.10.6"
-sha2 = "0.10.8"
+sha1 = "0.10.0"
+sha2 = "0.10.0"
 simd-json = "0.17.0"
 simdutf8 = "0.1.5"
 smallvec = "1.15.1"

README.md

@@ -15,10 +15,10 @@ An ergonomic all-in-one HTTP client for browser emulation with TLS, JA3/JA4, and
 
 | Crate | Version | Description |
 |-------|---------|-------------|
-| [`hpx`](https://crates.io/crates/hpx) | 2.4.4 | High Performance HTTP Client |
-| [`hpx-emulation`](https://crates.io/crates/hpx-emulation) | 2.4.4 | Browser emulation profiles |
-| [`hpx-yawc`](https://crates.io/crates/hpx-yawc) | 2.4.4 | WebSocket library (RFC 6455 + compression) |
-| [`hpx-fastwebsockets`](https://crates.io/crates/hpx-fastwebsockets) | 2.4.4 | Fast minimal WebSocket implementation |
+| [`hpx`](https://crates.io/crates/hpx) | 2.4.5 | High Performance HTTP Client |
+| [`hpx-emulation`](https://crates.io/crates/hpx-emulation) | 2.4.5 | Browser emulation profiles |
+| [`hpx-yawc`](https://crates.io/crates/hpx-yawc) | 2.4.5 | WebSocket library (RFC 6455 + compression) |
+| [`hpx-fastwebsockets`](https://crates.io/crates/hpx-fastwebsockets) | 2.4.5 | Fast minimal WebSocket implementation |
 
 ## Features
 
@@ -64,7 +64,7 @@ Add `hpx` to your `Cargo.toml`:
 
 ```toml
 [dependencies]
-hpx = "2.4.4"
+hpx = "2.4.5"
 ```
 
 The default features include **BoringSSL** TLS, **HTTP/1.1**, and **HTTP/2** support.
@@ -73,8 +73,8 @@ For browser emulation, add the utility crate:
 
 ```toml
 [dependencies]
-hpx = "2.4.4"
-hpx-emulation = "2.4.4"
+hpx = "2.4.5"
+hpx-emulation = "2.4.5"
 ```
 
 ## Feature Flags
@@ -125,7 +125,7 @@ The `ws` feature is an alias for `ws-yawc` (the default WebSocket backend). To u
 
 ```toml
 [dependencies]
-hpx = { version = "2.4.4", features = ["ws-fastwebsockets"] }
+hpx = { version = "2.4.5", features = ["ws-fastwebsockets"] }
 ```
 
 When both `ws-yawc` and `ws-fastwebsockets` are enabled, fastwebsockets takes priority.
@@ -135,37 +135,37 @@ When both `ws-yawc` and `ws-fastwebsockets` are enabled, fastwebsockets takes pr
 **Minimal HTTP client:**
 
 ```toml
-hpx = "2.4.4"  # default: boring + http1 + http2
+hpx = "2.4.5"  # default: boring + http1 + http2
 ```
 
 **JSON API client:**
 
 ```toml
-hpx = { version = "2.4.4", features = ["json", "cookies", "gzip"] }
+hpx = { version = "2.4.5", features = ["json", "cookies", "gzip"] }
 ```
 
 **WebSocket client:**
 
 ```toml
-hpx = { version = "2.4.4", features = ["ws"] }
+hpx = { version = "2.4.5", features = ["ws"] }
 ```
 
 **High-performance trading:**
 
 ```toml
-hpx = { version = "2.4.4", features = ["simd-json", "hickory-dns", "zstd", "ws"] }
+hpx = { version = "2.4.5", features = ["simd-json", "hickory-dns", "zstd", "ws"] }
 ```
 
 **Pure Rust (no C dependencies):**
 
 ```toml
-hpx = { version = "2.4.4", default-features = false, features = ["rustls-tls", "http1", "http2"] }
+hpx = { version = "2.4.5", default-features = false, features = ["rustls-tls", "http1", "http2"] }
 ```
 
 **Full-featured:**
 
 ```toml
-hpx = { version = "2.4.4", features = [
+hpx = { version = "2.4.5", features = [
     "json", "form", "query", "multipart", "stream",
     "cookies", "charset",
     "gzip", "brotli", "zstd", "deflate",
@@ -206,7 +206,7 @@ BoringSSL is the default TLS backend, providing robust support for modern TLS fe
 
 ```toml
 [dependencies]
-hpx = "2.4.4"
+hpx = "2.4.5"
 ```
 
 ### Rustls
@@ -217,7 +217,7 @@ A pure Rust TLS implementation. Useful for environments where C dependencies are
 
 ```toml
 [dependencies]
-hpx = { version = "2.4.4", default-features = false, features = ["rustls-tls", "http1", "http2"] }
+hpx = { version = "2.4.5", default-features = false, features = ["rustls-tls", "http1", "http2"] }
 ```
 
 ## Usage Examples

crates/hpx/Cargo.toml

@@ -22,7 +22,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 targets = ["x86_64-unknown-linux-gnu"]
 
 [features]
-default = ["boring", "http1", "http2"]
+default = ["boring", "http1", "http2", "stream", "tracing"]
 
 # Enable support for decoding text.
 charset = ["dep:encoding_rs", "dep:mime"]

crates/hpx/src/client.rs

@@ -27,7 +27,10 @@ pub use self::{
     conn::HttpInfo,
     core::upgrade::Upgraded,
     emulation::{BrowserProfile, Emulation, EmulationBuilder, EmulationFactory},
-    http::{Client, ClientBuilder},
+    http::{
+        Client, ClientBuilder, HttpVersionPreference, PoolConfigOptions, ProtocolConfigOptions,
+        ProxyConfigOptions, TlsConfigOptions, TransportConfigOptions,
+    },
     request::{Request, RequestBuilder},
     response::Response,
 };

crates/hpx/src/client/body.rs

@@ -1,3 +1,5 @@
+#![allow(clippy::items_after_test_module)]
+
 use std::{
     fmt,
     pin::Pin,

crates/hpx/src/client/core/conn/http1.rs

@@ -295,7 +295,16 @@ impl Builder {
         }
 
         let cd = proto::h1::dispatch::Client::new(rx);
-        let proto = proto::h1::Dispatcher::new(cd, conn);
+        let proto = proto::h1::Dispatcher::new_with_config(
+            cd,
+            conn,
+            proto::h1::dispatch::PollConfig {
+                max_iterations: self
+                    .opts
+                    .h1_max_poll_iterations
+                    .unwrap_or(proto::h1::dispatch::DEFAULT_MAX_POLL_ITERATIONS),
+            },
+        );
 
         Ok((SendRequest { dispatch: tx }, Connection { inner: proto }))
     }

crates/hpx/src/client/core/http1.rs

@@ -37,6 +37,9 @@ pub struct Http1Options {
 
     /// Whether to allow obsolete multiline headers in HTTP/1 responses.
     pub allow_obsolete_multiline_headers_in_responses: bool,
+
+    /// Maximum number of dispatcher loop iterations per scheduler poll.
+    pub h1_max_poll_iterations: Option<usize>,
 }
 
 impl Http1OptionsBuilder {
@@ -161,6 +164,20 @@ impl Http1OptionsBuilder {
         self
     }
 
+    /// Set the maximum number of dispatcher iterations per scheduler poll.
+    ///
+    /// The minimum value is 1.
+    #[inline]
+    pub fn max_poll_iterations(mut self, max_iterations: usize) -> Self {
+        assert!(
+            max_iterations > 0,
+            "max_poll_iterations must be greater than zero"
+        );
+
+        self.opts.h1_max_poll_iterations = Some(max_iterations);
+        self
+    }
+
     /// Build the [`Http1Options`] instance.
     #[inline]
     pub fn build(self) -> Http1Options {

crates/hpx/src/client/core/proto/h1/dispatch.rs

@@ -20,12 +20,30 @@ use crate::client::core::{
     upgrade::OnUpgrade,
 };
 
+/// Default number of HTTP/1 dispatcher iterations processed in a single poll.
+pub(crate) const DEFAULT_MAX_POLL_ITERATIONS: usize = 16;
+
+/// Configuration for the HTTP/1 dispatcher poll loop.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) struct PollConfig {
+    pub(crate) max_iterations: usize,
+}
+
+impl Default for PollConfig {
+    fn default() -> Self {
+        Self {
+            max_iterations: DEFAULT_MAX_POLL_ITERATIONS,
+        }
+    }
+}
+
 pub(crate) struct Dispatcher<D, Bs: Body, I, T> {
     conn: Conn<I, Bs::Data, T>,
     dispatch: D,
     body_tx: Option<body::Sender>,
     body_rx: Pin<Box<Option<Bs>>>,
     is_closing: bool,
+    poll_config: PollConfig,
 }
 
 pub(crate) trait Dispatch {
@@ -69,12 +87,21 @@ where
     Bs::Error: Into<BoxError>,
 {
     pub(crate) fn new(dispatch: D, conn: Conn<I, Bs::Data, T>) -> Self {
+        Self::new_with_config(dispatch, conn, PollConfig::default())
+    }
+
+    pub(crate) fn new_with_config(
+        dispatch: D,
+        conn: Conn<I, Bs::Data, T>,
+        poll_config: PollConfig,
+    ) -> Self {
         Dispatcher {
             conn,
             dispatch,
             body_tx: None,
             body_rx: Box::pin(None),
             is_closing: false,
+            poll_config,
         }
     }
 
@@ -129,9 +156,7 @@ where
         // Limit the looping on this connection, in case it is ready far too
         // often, so that other futures don't starve.
         //
-        // 16 was chosen arbitrarily, as that is number of pipelined requests
-        // benchmarks often use. Perhaps it should be a config option instead.
-        for _ in 0..16 {
+        for _ in 0..self.poll_config.max_iterations {
             let _ = self.poll_read(cx)?;
             let _ = self.poll_write(cx)?;
             let _ = self.poll_flush(cx)?;
@@ -148,6 +173,15 @@ where
                 // break;
                 return Poll::Ready(Ok(()));
             }
+
+            let mut consume_budget = std::pin::pin!(tokio::task::consume_budget());
+            if consume_budget.as_mut().poll(cx).is_pending() {
+                trace!(
+                    "poll_loop yielding due to cooperative budget (self = {:p})",
+                    self
+                );
+                return Poll::Pending;
+            }
         }
 
         trace!("poll_loop yielding (self = {:p})", self);
@@ -681,4 +715,18 @@ mod tests {
         // If it is, it will trigger an assertion.
         assert!(dispatcher.poll().is_pending());
     }
+
+    #[test]
+    fn dispatcher_uses_custom_poll_config() {
+        let io = tokio_test::io::Builder::new().build();
+        let (_tx, rx) = dispatch::channel();
+        let conn = Conn::<_, bytes::Bytes, ClientTransaction>::new(io);
+        let dispatcher = Dispatcher::new_with_config(
+            Client::<IncomingBody>::new(rx),
+            conn,
+            PollConfig { max_iterations: 7 },
+        );
+
+        assert_eq!(dispatcher.poll_config.max_iterations, 7);
+    }
 }

crates/hpx/src/client/core/proto/h1/io.rs

@@ -5,8 +5,8 @@ use std::{
     task::{Context, Poll, ready},
 };
 
-use bytes::{Buf, BufMut, Bytes, BytesMut};
-use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
+use bytes::{Buf, Bytes, BytesMut};
+use tokio::io::{AsyncRead, AsyncWrite};
 
 use super::{Http1Transaction, ParseContext, ParsedMessage};
 use crate::client::core::{self, Error, common::buf::BufList};
@@ -204,30 +204,22 @@ where
             self.read_buf.reserve(next);
         }
 
-        // SAFETY: ReadBuf and poll_read promise not to set any uninitialized
-        // bytes onto `dst`.
-        #[allow(unsafe_code)]
-        let dst = unsafe { self.read_buf.chunk_mut().as_uninit_slice_mut() };
-        let mut buf = ReadBuf::uninit(dst);
-        match Pin::new(&mut self.io).poll_read(cx, &mut buf) {
-            Poll::Ready(Ok(_)) => {
-                let n = buf.filled().len();
+        // Use tokio_util::io::poll_read_buf for safe buffer handling
+        // instead of manual unsafe operations
+        match tokio_util::io::poll_read_buf(Pin::new(&mut self.io), cx, &mut self.read_buf) {
+            Poll::Ready(Ok(n)) => {
                 trace!("received {} bytes", n);
-                #[allow(unsafe_code)]
-                unsafe {
-                    // Safety: we just read that many bytes into the
-                    // uninitialized part of the buffer, so this is okay.
-                    // @tokio pls give me back `poll_read_buf` thanks
-                    self.read_buf.advance_mut(n);
-                }
                 self.read_buf_strategy.record(n);
                 Poll::Ready(Ok(n))
             }
+            Poll::Ready(Err(e)) => {
+                debug!("read error: {}", e);
+                Poll::Ready(Err(e))
+            }
             Poll::Pending => {
                 self.read_blocked = true;
                 Poll::Pending
             }
-            Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
         }
     }