hpr_net/http.rs
1//! The HTTP transport: blocking HTTP/1.1 through `ureq`, TLS through rustls.
2
3use std::io::Read;
4use std::sync::Arc;
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::time::Duration;
7
8use ureq::{Proxy, ProxyProtocol};
9
10use crate::Transport;
11
12/// How an [`Http`] transport behaves. Start from [`HttpConfig::default`] and change fields.
13#[non_exhaustive]
14#[derive(Debug, Clone, PartialEq, Eq)]
15pub struct HttpConfig {
16 /// The time a whole request may take: finding the host, connecting, every redirect and
17 /// reading the body. Longer than [`Http::MAX_TIMEOUT`] counts as that. Default 60 s.
18 pub timeout: Duration,
19 /// The longest body accepted, counted after any gzip unpacking; one byte more is refused.
20 /// Default 64 MiB, room for a forecast grid and far past any JSON answer.
21 pub max_body_bytes: u64,
22 /// Whether to use the proxy the environment names (`ALL_PROXY`, `HTTPS_PROXY` or
23 /// `HTTP_PROXY`, the first set, in either case, for every URL, bar hosts in `NO_PROXY`).
24 /// Default `true`.
25 pub proxy_from_env: bool,
26}
27
28impl Default for HttpConfig {
29 fn default() -> Self {
30 Self {
31 timeout: Duration::from_secs(60),
32 max_body_bytes: 64 * 1024 * 1024,
33 proxy_from_env: true,
34 }
35 }
36}
37
38/// A [`Transport`] that fetches over HTTP and HTTPS, behind the crate's `http` feature.
39///
40/// TLS is rustls with Mozilla's root certificates compiled in (the `webpki-roots` crate), so it
41/// needs no OpenSSL and ignores the operating system's certificate store. It follows up to ten
42/// redirects and sends `Accept-Encoding: gzip`, unpacking gzip bodies. Any status but 2xx is an
43/// error, as are a timeout and a body that unpacks to more than the limit. An HTTP or HTTPS proxy
44/// from the environment is used. A SOCKS one is refused with an error rather than bypassed, since
45/// this build cannot speak SOCKS; hosts `NO_PROXY` exempts are fetched directly, with no redirect
46/// followed, since the next address might not be exempt.
47///
48/// Cloning is cheap and the clones share one pool of connections.
49///
50/// ```no_run
51/// use hpr_net::{Cache, Client, Http, Mode, Source};
52///
53/// let dir = Cache::platform_dir().expect("a home or local app data folder");
54/// let client = Client::new(Http::new(), Cache::new(dir), Mode::Online);
55/// let source = Source {
56/// name: "Example".into(),
57/// attribution: "Example data".into(),
58/// ttl_s: 3600,
59/// };
60/// let now_s = std::time::SystemTime::now()
61/// .duration_since(std::time::UNIX_EPOCH)
62/// .map_or(0, |d| d.as_secs());
63/// let answer = client.fetch(&source, "https://example.com/", now_s)?;
64/// println!("{} bytes, {:?}", answer.body.len(), answer.freshness);
65/// # Ok::<(), hpr_net::NetError>(())
66/// ```
67#[derive(Debug, Clone)]
68pub struct Http {
69 agent: ureq::Agent,
70 max_body_bytes: u64,
71 /// A SOCKS proxy the environment names, which every URL it covers is refused under.
72 socks: Option<Proxy>,
73 /// The body bytes the fetch in progress has read so far ([`Http::received`]).
74 received: Arc<AtomicU64>,
75}
76
77impl Http {
78 /// The `User-Agent` header sent: the package's name and version and the docs' address, so a
79 /// data provider can tell who is calling: `fusionspace-hpr/0.1.0 (+https://hpr.fusionspace.co)`
80 /// ([ADR-199][adr-199] ยง4).
81 ///
82 /// [adr-199]: https://github.com/nrdptel/fusionspace-eridanus/blob/main/docs/decisions/0199-the-2026-10-07-fusionspace-hpr.md
83 pub const USER_AGENT: &str = concat!(
84 "fusionspace-hpr/",
85 env!("CARGO_PKG_VERSION"),
86 " (+https://hpr.fusionspace.co)"
87 );
88 /// The longest timeout used: 30 days. A longer one, such as [`Duration::MAX`] for "none",
89 /// counts as this, since a deadline past the clock's range would overflow.
90 pub const MAX_TIMEOUT: Duration = Duration::from_secs(30 * 24 * 3600);
91
92 /// A transport with [`HttpConfig::default`].
93 pub fn new() -> Self {
94 Self::with_config(HttpConfig::default())
95 }
96
97 /// A transport with `config`.
98 pub fn with_config(config: HttpConfig) -> Self {
99 let proxy = if config.proxy_from_env {
100 Proxy::try_from_env()
101 } else {
102 None
103 };
104 Self::with_proxy(&config, proxy)
105 }
106
107 fn with_proxy(config: &HttpConfig, proxy: Option<Proxy>) -> Self {
108 let socks = proxy
109 .clone()
110 .filter(|p| !matches!(p.protocol(), ProxyProtocol::Http | ProxyProtocol::Https));
111 // Under a SOCKS proxy a redirect is refused, not followed: `NO_PROXY` could exempt the
112 // first address and not the next, which ureq would then reach directly.
113 let redirects = if socks.is_some() { 0 } else { 10 };
114 let agent = ureq::Agent::config_builder()
115 .timeout_global(Some(config.timeout.min(Self::MAX_TIMEOUT)))
116 .user_agent(Self::USER_AGENT)
117 .max_redirects(redirects)
118 .proxy(proxy)
119 .build()
120 .new_agent();
121 Self {
122 agent,
123 max_body_bytes: config.max_body_bytes,
124 socks,
125 received: Arc::default(),
126 }
127 }
128}
129
130/// The words a [`Transport::get`] refusal holds when the environment's proxy is SOCKS, which
131/// this transport doesn't speak: `reason.contains(SOCKS_REFUSED)` tells that refusal from a
132/// failed connection.
133pub const SOCKS_REFUSED: &str = "is SOCKS, which HPR Sim can't use";
134
135impl Http {
136 /// The body bytes, unpacked, that the fetch in progress has read so far, or the last fetch
137 /// read in all: set to 0 as a fetch starts and counted up as its body arrives, so another
138 /// thread can show a long fetch's progress while [`Transport::get`] waits. A clone of this
139 /// transport shares the count.
140 pub fn received(&self) -> Arc<AtomicU64> {
141 Arc::clone(&self.received)
142 }
143
144 /// Why `url` is refused, when the environment's proxy is SOCKS and `NO_PROXY` doesn't exempt
145 /// it. Without ureq's SOCKS support, ureq would warn and connect directly, around the proxy.
146 /// The reason holds [`SOCKS_REFUSED`]. A program can also leave the environment's proxy
147 /// unread ([`HttpConfig::proxy_from_env`]); the reason doesn't say so, as it reaches the
148 /// command line's users too.
149 fn socks_refusal(&self, url: &str) -> Option<String> {
150 let socks = self.socks.as_ref()?;
151 let uri = url.parse::<ureq::http::Uri>().ok()?;
152 if socks.is_no_proxy(&uri) {
153 return None;
154 }
155 // The protocol alone: the address may carry a user name and password, and one with an
156 // unescaped `/` or `#` in the password ends early, leaving the user name as the "host".
157 Some(format!(
158 "the environment's proxy ({:?}) {SOCKS_REFUSED}. Unset the variable naming it \
159 (ALL_PROXY, HTTPS_PROXY or HTTP_PROXY), or add the host to NO_PROXY",
160 socks.protocol(),
161 ))
162 }
163}
164
165impl Default for Http {
166 fn default() -> Self {
167 Self::new()
168 }
169}
170
171impl Transport for Http {
172 fn get(&self, url: &str) -> Result<Vec<u8>, String> {
173 self.received.store(0, Ordering::Relaxed);
174 if let Some(refusal) = self.socks_refusal(url) {
175 return Err(refusal);
176 }
177 let mut response = self.agent.get(url).call().map_err(|e| e.to_string())?;
178 // ureq errors on 4xx and 5xx only; a 304 or an unfollowed 3xx would read as an empty body.
179 let status = response.status();
180 if !status.is_success() {
181 return Err(format!("http status: {}", status.as_u16()));
182 }
183 // The limit is on the unpacked body, read one byte past it to tell "at" from "over".
184 // ureq's own limit counts bytes on the wire, inside its gzip decoder, so a small compressed
185 // body could unpack past it. Wire bytes that unpack to nothing are bounded by the timeout.
186 let unpacked = response.body_mut().with_config().limit(u64::MAX).reader();
187 let mut body = Vec::new();
188 Counted {
189 inner: unpacked,
190 received: &self.received,
191 }
192 .take(self.max_body_bytes.saturating_add(1))
193 .read_to_end(&mut body)
194 .map_err(|e| e.to_string())?;
195 if body.len() as u64 > self.max_body_bytes {
196 return Err(format!(
197 "the body is longer than the {} bytes allowed",
198 self.max_body_bytes
199 ));
200 }
201 Ok(body)
202 }
203}
204
205/// A reader that adds each read's length to `received`.
206struct Counted<'a, R> {
207 inner: R,
208 received: &'a AtomicU64,
209}
210
211impl<R: Read> Read for Counted<'_, R> {
212 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
213 let read = self.inner.read(buf)?;
214 self.received.fetch_add(read as u64, Ordering::Relaxed);
215 Ok(read)
216 }
217}
218
219#[cfg(test)]
220mod tests {
221 use super::*;
222
223 /// The count is each read's length, added up, whatever the reads' sizes.
224 #[test]
225 fn a_counted_reader_adds_up_what_it_reads() {
226 let received = AtomicU64::new(0);
227 let mut body = Vec::new();
228 Counted {
229 inner: &[7_u8; 10_000][..],
230 received: &received,
231 }
232 .read_to_end(&mut body)
233 .unwrap();
234 assert_eq!(body.len(), 10_000);
235 assert_eq!(received.load(Ordering::Relaxed), 10_000);
236 }
237
238 /// A fetch starts its count from 0, even one refused before it reaches the network.
239 #[test]
240 fn a_fetch_starts_its_count_from_zero() {
241 let http = under(Proxy::new("socks5://127.0.0.1:9").unwrap());
242 http.received().store(99, Ordering::Relaxed);
243 assert!(http.get("http://127.0.0.1:1/x").is_err());
244 assert_eq!(http.received().load(Ordering::Relaxed), 0);
245 }
246
247 fn under(proxy: Proxy) -> Http {
248 Http::with_proxy(&HttpConfig::default(), Some(proxy))
249 }
250
251 #[test]
252 fn a_socks_proxy_is_refused_not_bypassed() {
253 let http = under(Proxy::new("socks5://127.0.0.1:9").unwrap());
254 let reason = http.get("http://127.0.0.1:1/x").unwrap_err();
255 assert!(reason.contains("is SOCKS"), "{reason}");
256 }
257
258 #[test]
259 fn the_refusal_never_repeats_the_proxys_credentials() {
260 // Well formed, and with an unescaped `/` that ends the address inside the password.
261 for url in [
262 "socks5://alice:s3cret@proxy.example:1080",
263 "socks5://alice:12/cret@proxy.example:1080",
264 ] {
265 let reason = under(Proxy::new(url).unwrap())
266 .socks_refusal("https://example.test/x")
267 .unwrap();
268 assert!(
269 !reason.contains("cret") && !reason.contains("alice"),
270 "{reason}"
271 );
272 assert!(reason.contains(SOCKS_REFUSED), "{reason}");
273 }
274 }
275
276 #[test]
277 fn no_proxy_exempts_a_host_from_the_refusal() {
278 let proxy = Proxy::builder(ProxyProtocol::Socks5)
279 .host("proxy.example")
280 .no_proxy("near.example")
281 .build()
282 .unwrap();
283 let http = under(proxy);
284 assert_eq!(http.socks_refusal("https://near.example/x"), None);
285 assert!(http.socks_refusal("https://far.example/x").is_some());
286 }
287
288 #[test]
289 fn under_a_socks_proxy_redirects_are_not_followed() {
290 // A request can't be made here: ureq panics on a SOCKS proxy not read from the
291 // environment, and only the environment makes one. With no redirects followed, a 3xx
292 // comes back as the response and fails as any status but 2xx does (`tests/http.rs`, 304).
293 let socks = under(Proxy::new("socks5://127.0.0.1:9").unwrap());
294 assert_eq!(socks.agent.config().max_redirects(), 0);
295 let http = under(Proxy::new("http://127.0.0.1:9").unwrap());
296 assert_eq!(http.agent.config().max_redirects(), 10);
297 }
298
299 #[test]
300 fn an_http_proxy_is_used_not_refused() {
301 for url in ["http://127.0.0.1:9", "https://127.0.0.1:9"] {
302 let http = under(Proxy::new(url).unwrap());
303 assert_eq!(http.socks_refusal("https://example.test/x"), None, "{url}");
304 }
305 }
306}