Avoid file descriptor exhaustion when fetching certificates.

- Previously, there was a chance of running out of file descriptors
    while or after fetching a large number of certificates using sq
    network fetch.

  - The root cause of that was the use of getaddrinfo(3) to resolve
    names, which is a blocking interface, which has to be executed on
    a special thread for blocking tasks on the tokio runtime.  The
    maximum number of these threads is capped at 512 by default, and
    these threads can tie up a significant number of file descriptors
    in sockets.  The threads do close their sockets and go away after
    a while, presumably after a timeout.  Further, blocking tasks can
    not be canceled.

  - Do release all thread pool resources after doing the fetch.

  - Also, switch to the hickory crate for doing name lookups.  This
    implements a non-blocking interface, and releases resources in a
    timely fashion.

  - Fixes #335.
This commit is contained in:
Justus Winter 2024-09-27 13:34:31 +02:00
parent 8468817010
commit f448fcb347
No known key found for this signature in database
GPG Key ID: 686F55B4AB2B3386
3 changed files with 6 additions and 0 deletions

2
Cargo.lock generated
View File

@ -3019,6 +3019,7 @@ dependencies = [
"futures-core", "futures-core",
"futures-util", "futures-util",
"h2", "h2",
"hickory-resolver",
"http", "http",
"http-body", "http-body",
"hyper", "hyper",
@ -3572,6 +3573,7 @@ dependencies = [
"once_cell", "once_cell",
"predicates", "predicates",
"regex", "regex",
"reqwest",
"roff", "roff",
"rpassword", "rpassword",
"sequoia-autocrypt", "sequoia-autocrypt",

View File

@ -43,6 +43,7 @@ clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] }
humantime = "2" humantime = "2"
indicatif = "0.17" indicatif = "0.17"
once_cell = "1.17" once_cell = "1.17"
reqwest = { version = "0.11.27", features = ["hickory-dns"] }
sequoia-cert-store = "0.6.0" sequoia-cert-store = "0.6.0"
sequoia-keystore = { version = ">=0.5, <0.7" } sequoia-keystore = { version = ">=0.5, <0.7" }
sequoia-wot = { version = "0.12", default-features = false } sequoia-wot = { version = "0.12", default-features = false }

View File

@ -857,6 +857,9 @@ pub fn dispatch_fetch(mut sq: Sq, c: cli::network::fetch::Command)
})?; })?;
drop(pb); drop(pb);
// Release all thread pool resources.
drop(rt);
Response::import_or_emit(sq, c.output, c.binary, results)?; Response::import_or_emit(sq, c.output, c.binary, results)?;
Ok(()) Ok(())
} }