app: Add support for passing URLs to RPMs

This teaches the client to fetch packages from URLs directly so that one
doesn't have to `curl` first and then install. Supported anywhere
package filenames are allowed (notably: `install` and
`override replace`).

One neat things about this is that we download the file into an
`O_TMPFILE` and then pass on ownership of that fd directly to the
daemon. So at no point are the packages actually laying visible on the
system. (Assuming the filesystem supports `O_TMPFILE` that is).

This adds direct linking to libcurl and openssl, two libraries which we
were already pulling in indirectly.

Closes: #1508
Approved by: cgwalters

Makefile-rpm-ostree.am

@@ -99,9 +99,9 @@ $(librpmostree_rust_path): Makefile $(LIBRPMOSTREE_RUST_SRCS)
 	  $(cargo) build --verbose $${frozen} $(CARGO_RELEASE_ARGS)
 EXTRA_DIST += $(LIBRPMOSTREE_RUST_SRCS) rust/Cargo.lock
 
-rpm_ostree_CFLAGS += -Irust/include
+rpm_ostree_CFLAGS += -Irust/include $(PKGDEP_RPMOSTREE_RS_CFLAGS)
 rpm_ostree_SOURCES += rust/include/librpmostree-rust.h
-rpm_ostree_LDADD += $(librpmostree_rust_path)
+rpm_ostree_LDADD += $(librpmostree_rust_path) $(PKGDEP_RPMOSTREE_RS_LIBS)
 rustfmt:
 	rustfmt $(LIBRPMOSTREE_RUST_SRCS)
 # Outside the ifdef, otherwise automake complains

configure.ac

@@ -99,6 +99,13 @@ PKG_CHECK_MODULES(PKGDEP_RPMOSTREE, [gio-unix-2.0 >= 2.50.0 json-glib-1.0
 				     polkit-gobject-1
 				     rpm librepo libsolv
 				     libarchive])
+
+# We just keep rust-specific deps separate for better tracking
+# The `libcurl` one is redundant since we already require it for libostree. `openssl`
+# is required by libcurl anyway, but we need to link to it directly too because
+# curl-rust uses it.
+PKG_CHECK_MODULES(PKGDEP_RPMOSTREE_RS, [libcurl openssl])
+
 dnl bundled libdnf
 PKGDEP_RPMOSTREE_CFLAGS="-I $(pwd)/libdnf -I $(pwd)/libdnf-build $PKGDEP_RPMOSTREE_CFLAGS"
 PKGDEP_RPMOSTREE_LIBS="-L$(pwd)/libdnf-build/libdnf -ldnf $PKGDEP_RPMOSTREE_LIBS"

rust/Cargo.toml

@@ -14,6 +14,7 @@ gio-sys = "0.6.0"
 glib = "0.5.0"
 tempfile = "3.0.3"
 openat = "0.1.15"
+curl = "0.4.14"
 
 [lib]
 name = "rpmostree_rust"

rust/include/rpmostree-rust.h

@@ -33,3 +33,5 @@ const char *rpmostree_rs_treefile_get_rojig_spec_path (RpmOstreeRsTreefile *tf);
 
 void rpmostree_rs_treefile_free (RpmOstreeRsTreefile *tf);
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(RpmOstreeRsTreefile, rpmostree_rs_treefile_free);
+
+int rpmostree_rs_download_to_fd (const char *url, GError **error);

rust/src/lib.rs

@@ -22,6 +22,7 @@ extern crate glib_sys;
 extern crate libc;
 extern crate openat;
 extern crate tempfile;
+extern crate curl;
 
 #[macro_use]
 extern crate serde_derive;
@@ -38,6 +39,7 @@ mod glibutils;
 use glibutils::*;
 mod treefile;
 use treefile::*;
+mod utils;
 
 /* Wrapper functions for translating from C to Rust */
 
@@ -129,3 +131,18 @@ pub extern "C" fn rpmostree_rs_treefile_free(tf: *mut Treefile) {
         Box::from_raw(tf);
     }
 }
+
+#[no_mangle]
+pub extern "C" fn rpmostree_rs_download_to_fd(
+    url: *const libc::c_char,
+    gerror: *mut *mut glib_sys::GError,
+) -> libc::c_int {
+    let url = str_from_nullable(url).unwrap();
+    match utils::download_url_to_tmpfile(url) {
+        Ok(f) => f.into_raw_fd() as libc::c_int,
+        Err(e) => {
+            error_to_glib(&e, gerror);
+            -1 as libc::c_int
+        }
+    }
+}

rust/src/utils.rs

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+use std::{fs, io};
+use std::io::prelude::*;
+use tempfile;
+
+use curl::easy::Easy;
+
+pub fn download_url_to_tmpfile(url: &str) -> io::Result<fs::File> {
+    let mut tmpf = tempfile::tempfile()?;
+    {
+        let mut output = io::BufWriter::new(&mut tmpf);
+        let mut handle = Easy::new();
+        handle.follow_location(true)?;
+        handle.fail_on_error(true)?;
+        handle.url(url)?;
+
+        let mut transfer = handle.transfer();
+        transfer.write_function(|data| {
+            output.write_all(data).and(Ok(data.len())).or(Ok(0))
+        })?;
+        transfer.perform()?;
+    }
+
+    tmpf.seek(io::SeekFrom::Start(0))?;
+    Ok(tmpf)
+}

src/app/rpmostree-dbus-helpers.c

@@ -33,6 +33,7 @@
 #include "rpmostree-libbuiltin.h"
 #include "rpmostree-util.h"
 #include "rpmostree-rpm-util.h"
+#include "rpmostree-rust.h"
 
 #define RPMOSTREE_CLI_ID "cli"
 
@@ -1042,7 +1043,25 @@ rpmostree_sort_pkgs_strv (const char *const* pkgs,
   g_variant_builder_init (&builder, G_VARIANT_TYPE ("ah"));
   for (const char *const* pkg = pkgs; pkg && *pkg; pkg++)
     {
-      if (!g_str_has_suffix (*pkg, ".rpm"))
+      if (g_str_has_prefix (*pkg, "http://") ||
+          g_str_has_prefix (*pkg, "https://"))
+        {
+          g_print ("Downloading '%s'... ", *pkg);
+          glnx_autofd int fd = rpmostree_rs_download_to_fd (*pkg, error);
+          if (fd < 0)
+            {
+              g_print ("failed!\n");
+              return FALSE;
+            }
+          g_print ("done!\n");
+
+          int idx = g_unix_fd_list_append (fd_list, fd, error);
+          if (idx < 0)
+            return FALSE;
+
+          g_variant_builder_add (&builder, "h", idx);
+        }
+      else if (!g_str_has_suffix (*pkg, ".rpm"))
         g_ptr_array_add (repo_pkgs, g_strdup (*pkg));
       else
         {

tests/vmcheck/test-override-kernel.sh

@@ -32,18 +32,15 @@ fi
 
 # Test that we can override the kernel.  For ease of testing
 # I just picked the "gold" F28 kernel.
-vm_cmd 'curl -sS -L \
-  -O https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Everything/x86_64/os/Packages/k/kernel-4.16.3-301.fc28.x86_64.rpm \
-  -O https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Everything/x86_64/os/Packages/k/kernel-core-4.16.3-301.fc28.x86_64.rpm \
-  -O https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Everything/x86_64/os/Packages/k/kernel-modules-4.16.3-301.fc28.x86_64.rpm \
-  -O https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Everything/x86_64/os/Packages/k/kernel-modules-extra-4.16.3-301.fc28.x86_64.rpm'
 current=$(vm_get_booted_csum)
 vm_cmd rpm-ostree db list "${current}" > current-dblist.txt
 assert_not_file_has_content current-dblist.txt 'kernel-4.16.3-301.fc28'
 grep -E '^ kernel-4' current-dblist.txt  | sed -e 's,^ *,,' > orig-kernel.txt
 assert_streq "$(wc -l < orig-kernel.txt)" "1"
 orig_kernel=$(cat orig-kernel.txt)
-vm_rpmostree override replace ./kernel*4.16.3*.rpm
+URL_ROOT="https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Everything/x86_64/os/Packages/k"
+vm_rpmostree override replace \
+  "$URL_ROOT/kernel{,-core,-modules{,-extra}}-4.16.3-301.fc28.x86_64.rpm"
 new=$(vm_get_pending_csum)
 vm_cmd rpm-ostree db list "${new}" > new-dblist.txt
 assert_file_has_content_literal new-dblist.txt 'kernel-4.16.3-301.fc28'