Avoid file descriptor exhaustion when fetching certificates.

- Previously, there was a chance of running out of file descriptors
    while or after fetching a large number of certificates using sq
    network fetch.

  - The root cause of that was the use of getaddrinfo(3) to resolve
    names, which is a blocking interface, which has to be executed on
    a special thread for blocking tasks on the tokio runtime.  The
    maximum number of these threads is capped at 512 by default, and
    these threads can tie up a significant number of file descriptors
    in sockets.  The threads do close their sockets and go away after
    a while, presumably after a timeout.  Further, blocking tasks can
    not be canceled.

  - Do release all thread pool resources after doing the fetch.

  - Also, switch to the hickory crate for doing name lookups.  This
    implements a non-blocking interface, and releases resources in a
    timely fashion.

  - Fixes #335.
This commit is contained in:
Justus Winter 2024-09-27 13:34:31 +02:00
parent 8468817010
commit f448fcb347
No known key found for this signature in database
GPG Key ID: 686F55B4AB2B3386
3 changed files with 6 additions and 0 deletions

2
Cargo.lock generated
View File

@ -3019,6 +3019,7 @@ dependencies = [
"futures-core",
"futures-util",
"h2",
"hickory-resolver",
"http",
"http-body",
"hyper",
@ -3572,6 +3573,7 @@ dependencies = [
"once_cell",
"predicates",
"regex",
"reqwest",
"roff",
"rpassword",
"sequoia-autocrypt",

View File

@ -43,6 +43,7 @@ clap = { version = "4", features = ["derive", "env", "string", "wrap_help"] }
humantime = "2"
indicatif = "0.17"
once_cell = "1.17"
reqwest = { version = "0.11.27", features = ["hickory-dns"] }
sequoia-cert-store = "0.6.0"
sequoia-keystore = { version = ">=0.5, <0.7" }
sequoia-wot = { version = "0.12", default-features = false }

View File

@ -857,6 +857,9 @@ pub fn dispatch_fetch(mut sq: Sq, c: cli::network::fetch::Command)
})?;
drop(pb);
// Release all thread pool resources.
drop(rt);
Response::import_or_emit(sq, c.output, c.binary, results)?;
Ok(())
}