tests/ext/destructive: enhance test logic

This enhances external-tests logic, ensuring that destructive tests
have retries and some context to pinpoint failures, and that failed-state
services are reset between iterations.

tests/inst/src/destructive.rs

@@ -393,6 +393,8 @@ fn impl_transaction_test<M: AsRef<str>>(
     // then we'll exit implicitly via the reboot, and reenter the function
     // above.
     loop {
+        // Make sure previously failed services (if any) can run.
+        bash!("systemctl reset-failed || true")?;
         // Save the previous strategy as a string so we can use it in error
         // messages below
         let prev_strategy_str = format!("{:?}", live_strategy);

tests/inst/src/rpmostree.rs

@@ -1,7 +1,6 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use serde_derive::Deserialize;
-use serde_json;
-use std::process::{Command, Stdio};
+use std::process::Command;
 
 #[derive(Deserialize)]
 #[serde(rename_all = "kebab-case")]
@@ -25,9 +24,29 @@ pub(crate) struct Deployment {
 }
 
 pub(crate) fn query_status() -> Result<Status> {
-    let cmd = Command::new("rpm-ostree")
-        .args(&["status", "--json"])
-        .stdout(Stdio::piped())
-        .spawn()?;
-    Ok(serde_json::from_reader(cmd.stdout.unwrap())?)
+    // Retry on temporary activation failures, see
+    // https://github.com/coreos/rpm-ostree/issues/2531
+    let pause = std::time::Duration::from_secs(1);
+    let mut retries = 0;
+    let cmd_res = loop {
+        retries += 1;
+        let res = Command::new("rpm-ostree")
+            .args(&["status", "--json"])
+            .output()
+            .context("failed to spawn 'rpm-ostree status'")?;
+
+        if res.status.success() || retries >= 10 {
+            break res;
+        }
+        std::thread::sleep(pause);
+    };
+
+    if !cmd_res.status.success() {
+        anyhow::bail!(
+            "running 'rpm-ostree status' failed: {}",
+            String::from_utf8_lossy(&cmd_res.stderr)
+        );
+    }
+
+    serde_json::from_slice(&cmd_res.stdout).context("failed to parse 'rpm-ostree status' output")
 }