Implement sq network wkd publish.

Cargo.lock

@@ -1291,6 +1291,12 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
 [[package]]
 name = "futures"
 version = "0.3.30"
@@ -3582,6 +3588,7 @@ dependencies = [
  "dirs",
  "dot-writer",
  "fehler",
+ "fs_extra",
  "humantime",
  "indicatif",
  "itertools",

Cargo.toml

@@ -32,6 +32,7 @@ maintenance = { status = "actively-developed" }
 buffered-reader = { version = "1.3.1", default-features = false, features = ["compression"] }
 dirs = "5"
 dot-writer = { version = "0.1.3", optional = true }
+fs_extra = "1"
 sequoia-directories = "0.1"
 sequoia-openpgp = { version = "1.18", default-features = false, features = ["compression"] }
 sequoia-autocrypt = { version = "0.25", default-features = false }

NEWS

@@ -103,6 +103,9 @@
 
    - When showing a user ID for a certificate, choose the one that is
      most authenticated.
+
+   - `sq network wkd publish` publishes and updates WKD hierarchies
+     via rsync.
 * Changes in 0.33.0
 ** Notable changes
    - The command line interface has been restructured.  Please consult

src/cli/network/wkd.rs

@@ -2,11 +2,18 @@ use std::path::PathBuf;
 
 use clap::{Args, Parser, Subcommand};
 
+use sequoia_net::wkd;
+
 use crate::cli::types::ClapData;
 use crate::cli::types::FileOrCertStore;
 use crate::cli::types::FileOrStdin;
 use crate::cli::types::FileOrStdout;
 
+use crate::cli::examples;
+use examples::Action;
+use examples::Actions;
+use examples::Example;
+
 #[derive(Parser, Debug)]
 #[clap(
     name = "wkd",
@@ -29,6 +36,7 @@ pub struct Command {
 pub enum Subcommands {
     Fetch(FetchCommand),
     Generate(GenerateCommand),
+    Publish(PublishCommand),
     DirectUrl(DirectUrlCommand),
     Url(UrlCommand),
 }
@@ -164,3 +172,113 @@ pub struct GenerateCommand {
     )]
     pub skip: bool,
 }
+
+const PUBLISH_EXAMPLES: Actions = Actions {
+    actions: &[
+        Action::Example(Example {
+            comment: "Create a new WKD hierarchy in the local directory \
+                      `public_html`, and insert Alice's cert.",
+            command: &[
+                "sq", "network", "wkd", "publish", "--create",
+                "--cert", "EB28F26E2739A4870ECC47726F0073F60FD0CBF0",
+                "example.org", "public_html",
+            ],
+        }),
+
+        Action::Example(Example {
+            comment: "Add Bob's cert to the existing WKD hierarchy \
+                      in the local directory `public_html`.",
+            command: &[
+                "sq", "network", "wkd", "publish",
+                "--cert", "511257EBBF077B7AEDAE5D093F68CB84CE537C9A",
+                "example.org", "public_html",
+            ],
+        }),
+
+        Action::Example(Example {
+            comment: "Refresh all certs in the existing WKD hierarchy \
+                      in the local directory `public_html` from the \
+                      cert store.",
+            command: &[
+                "sq", "network", "wkd", "publish",
+                "example.org", "public_html",
+            ],
+        }),
+    ],
+};
+
+test_examples!(sq_network_wkd_publish, PUBLISH_EXAMPLES);
+
+#[derive(Debug, Args)]
+#[clap(
+    about = "Publish certificates in a Web Key Directory",
+    long_about =
+"Publish certificates in a Web Key Directory
+
+Publishes certificates or certificate updates in a Web Key Directory
+(WKD).  You can create or update a WKD hierarchy on the local system by
+specifying a path as destination.
+
+Typically, a WKD is stored on a web server.  If --rsync is given, this
+command manages remote WKD directory hierarchies by using rsync(1).
+
+To update a WKD hierarchy, it is first copied to a temporary location
+on the local machine, new certificates or certificate updates are
+inserted into the local copy, and the hierarchy is copied back to its
+original location.  As this is not an atomic operation, care must be
+taken to avoid concurrent updates.
+",
+    after_help = PUBLISH_EXAMPLES,
+)]
+pub struct PublishCommand {
+    #[clap(
+        long = "create",
+        value_name = "METHOD",
+        default_missing_value = "advanced",
+        num_args = 0..=1,
+        help = "Create the WKD hierarchy if it does not exist yet",
+    )]
+    pub create: Option<Method>,
+    #[clap(
+        long = "cert",
+        value_name = "FINGERPRINT",
+        help = "Insert the given cert into the WKD",
+    )]
+    pub certs: Vec<String>,
+    #[clap(
+        long = "rsync",
+        value_name = "RSYNC",
+        default_missing_value = "rsync",
+        num_args = 0..=1,
+        help = "Path to the local rsync command to use",
+    )]
+    pub rsync: Option<String>,
+    #[clap(
+        value_name = "FQDN",
+        help = "Generate a WKD for a fully qualified domain name for email",
+    )]
+    pub domain: String,
+    #[clap(
+        value_name = "DEST",
+        help = "WKD location on the server, passed to rsync(1)",
+        long_help = "Location of the WKD hierarchy on the local machine or \
+                     a remote server.  If --rsync is given, this is passed \
+                     as-is to rsync(1).",
+    )]
+    pub destination: String,
+}
+
+#[derive(clap::ValueEnum, Debug, Clone)]
+pub enum Method {
+    Advanced,
+    Direct,
+}
+
+impl From<Method> for wkd::Variant {
+    fn from(v: Method) -> wkd::Variant {
+        match v {
+            Method::Advanced => wkd::Variant::Advanced,
+            Method::Direct => wkd::Variant::Direct,
+        }
+    }
+}

src/commands/network.rs

@@ -2,6 +2,8 @@
 
 use std::collections::HashSet;
 use std::fmt;
+use std::fs::{self, DirEntry};
+use std::path::Path;
 use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
@@ -911,6 +913,7 @@ pub fn dispatch_keyserver(mut sq: Sq,
             Response::import_or_emit(sq, c.output, c.binary, certs)?;
             Result::Ok(())
         })?,
+
         Publish(c) => rt.block_on(async {
             let mut input = c.input.open()?;
             let cert = Arc::new(Cert::from_buffered_reader(&mut input).
@@ -1058,8 +1061,164 @@ pub fn dispatch_wkd(mut sq: Sq, c: cli::network::wkd::Command)
                 }
             }
         },
+
+        Publish(c) => {
+            use wkd::Variant;
+            let cert_store = sq.cert_store_or_else()?;
+            let insert = c.certs.iter()
+                .map(|fp| {
+                    let fp = fp.parse().with_context(
+                        || format!("Parsing {:?} as fingerprint", fp))?;
+                    cert_store.lookup_by_cert_fpr(&fp)
+                        .with_context(
+                            || format!("Looking up {} for insertion", fp))
+                })
+                .collect::<Result<Vec<_>>>()?;
+
+            // Strategy: We transfer the WKD to a temporary directory,
+            // read all the certs, update them from the local cert
+            // store, re-create the WKD hierarchy, then transfer it
+            // back.
+            let wd = tempfile::TempDir::new()?;
+
+            // First, fetch the WKD.
+            let fetch = wd.path().join("fetch");
+            fs::create_dir(&fetch)?;
+            let r = transfer(&c.rsync,
+                  &format!("{}/.well-known/openpgpkey", c.destination),
+                  &fetch.display().to_string())
+                .context("failed to copy the remote WKD hierarchy \
+                          to the local system");
+            if r.is_err() && c.create.is_none() {
+                return r;
             }
 
+            // Detect the variant by locating the policy file.
+            let fetch = fetch.join("openpgpkey");
+            let direct_policy = fetch.join("policy");
+            let advanced_policy = fetch.join(&c.domain).join("policy");
+            let (variant, policy) = match (direct_policy.exists(),
+                                           advanced_policy.exists())
+            {
+                (true, false) => (Variant::Direct, Some(direct_policy)),
+                (false, true) => (Variant::Advanced, Some(advanced_policy)),
+                (false, false) => if let Some(m) = c.create {
+                    (m.into(), None)
+                } else {
+                    return Err(anyhow::anyhow!("No policy file found")
+                               .context("Neither direct nor advanced \
+                                         WKD detected"))
+                },
+                (true, true) =>
+                    return Err(anyhow::anyhow!("Two policy files found")
+                               .context("Both direct and advanced \
+                                         WKD detected")),
+            };
+            let hu = match variant {
+                Variant::Direct => fetch.join("hu"),
+                Variant::Advanced => fetch.join(&c.domain).join("hu"),
+            };
+
+            // Now re-create the WKD hierarchy while updating the certs.
+            let push = wd.path().join("push");
+            let push_wk = push.join(".well-known");
+            let push_openpgpkey = push_wk.join("openpgpkey");
+            fs::create_dir(&push)?;
+            visit_dirs(&hu, &|entry: &DirEntry| -> Result<()> {
+                let p = entry.path();
+                for cert in CertParser::from_reader(fs::File::open(p)?)? {
+                    let mut cert = cert?;
+                    if let Ok(update) =
+                        cert_store.lookup_by_cert_fpr(&cert.fingerprint())
+                    {
+                        cert = cert.merge_public(update.to_cert()?.clone())?;
+                    }
+
+                    wkd::insert(&push, &c.domain, variant, &cert)?;
+                }
+                Ok(())
+            })?;
+
+            // Insert the new ones, if any.
+            for cert in insert {
+                wkd::insert(&push, &c.domain, variant, cert.to_cert()?)?;
+            }
+
+            // Preserve the original policy file, if any.
+            if let Some(policy) = policy {
+                match variant {
+                    Variant::Direct => fs::copy(
+                        policy,
+                        push_openpgpkey.join("policy"))?,
+                    Variant::Advanced => fs::copy(
+                        policy,
+                        push_openpgpkey.join(&c.domain).join("policy"))?,
+                };
+            }
+
+            // Finally, transfer the WKD hierarchy back.
+            transfer(&c.rsync, &push_wk.display().to_string(),
+                     &format!("{}", c.destination))
+                .context("failed to copy the local WKD hierarchy \
+                          to the remote system")?;
+        },
+    }
+
+    Ok(())
+}
+
+fn transfer(rsync_bin: &Option<String>, source: &str, destination: &str)
+            -> Result<()>
+{
+    if let Some(r) = rsync_bin {
+        rsync(r, source, destination)
+    } else {
+        copy(source, destination)
+    }
+}
+
+fn copy(source: &str, destination: &str) -> Result<()> {
+    let options = fs_extra::dir::CopyOptions::new()
+        .overwrite(true)
+    //.content_only(true)
+    //.copy_inside(true)
+        ;
+
+    std::fs::create_dir_all(destination)?;
+    fs_extra::dir::copy(source, destination, &options)?;
+    Ok(())
+}
+
+fn rsync(rsync: &str, source: &str, destination: &str) -> Result<()> {
+    use std::process::Command;
+
+    let status = Command::new(rsync)
+        .arg("--recursive")
+        .arg(source)
+        .arg(destination)
+        .spawn()?
+        .wait()?;
+
+    if status.success() {
+        Ok(())
+    } else {
+        Err(anyhow::anyhow!("rsync failed"))
+    }
+}
+
+// one possible implementation of walking a directory only visiting files
+fn visit_dirs(dir: &Path, cb: &dyn Fn(&DirEntry) -> Result<()>) -> Result<()> {
+    if dir.is_dir() {
+        for entry in fs::read_dir(dir)? {
+            let entry = entry?;
+            let path = entry.path();
+            if path.is_dir() {
+                visit_dirs(&path, cb)?;
+            } else {
+                cb(&entry)?;
+            }
+        }
+    }
     Ok(())
 }