shlib: Make basearch APIs call /usr/bin/rpm-ostree shlib-backend

This is a potential path to fix
https://github.com/coreos/rpm-ostree/issues/2391

Basically our shared library and executable duplicate too much
code (particularly Rust).  Since most of the shlib APIs aren't
performance sensitive, let's have them fork off the binary
via a hidden CLI entrypoint and parse the output.

Makefile-rpm-ostree.am

@@ -42,6 +42,7 @@ rpm_ostree_SOURCES = src/app/main.cxx \
 	src/app/rpmostree-builtin-status.c \
 	src/app/rpmostree-builtin-ex.c \
 	src/app/rpmostree-builtin-testutils.c \
+	src/app/rpmostree-builtin-shlib-backend.c \
 	src/app/rpmostree-builtin-db.c \
 	src/app/rpmostree-builtin-start-daemon.c \
 	src/app/rpmostree-builtin-finalize-deployment.c \

src/app/main.cxx

@@ -119,6 +119,9 @@ static RpmOstreeCommand commands[] = {
   { "testutils", static_cast<RpmOstreeBuiltinFlags>(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD |
                  RPM_OSTREE_BUILTIN_FLAG_HIDDEN),
     NULL, rpmostree_builtin_testutils },
+  { "shlib-backend", static_cast<RpmOstreeBuiltinFlags>(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD |
+                 RPM_OSTREE_BUILTIN_FLAG_HIDDEN),
+    NULL, rpmostree_builtin_shlib_backend },
   { "start-daemon", static_cast<RpmOstreeBuiltinFlags>(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD |
                     RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT |
                     RPM_OSTREE_BUILTIN_FLAG_HIDDEN),

src/app/rpmostree-builtin-shlib-backend.c

@@ -0,0 +1,107 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2020 Red Hat, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; either version 2 of the licence or (at
+ * your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General
+ * Public License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place, Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <string.h>
+#include <glib-unix.h>
+#include <gio/gio.h>
+#include <gio/gunixfdmessage.h>
+#include <gio/gunixsocketaddress.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include "rpmostree-builtins.h"
+#include "rpmostree-libbuiltin.h"
+#include "rpmostree-rust.h"
+#include "rpmostree.h"
+#include "rpmostree-core.h"
+#include "src/lib/rpmostree-shlib-ipc-private.h"
+
+#include <libglnx.h>
+
+static gboolean
+send_memfd_result (int ret_memfd, GError **error)
+{
+  int fdarray[] = {ret_memfd, -1 };
+  g_autoptr(GUnixFDList) list = g_unix_fd_list_new_from_array (fdarray, 1);
+  g_autoptr(GUnixFDMessage) message = G_UNIX_FD_MESSAGE (g_unix_fd_message_new_with_fd_list (list));
+  g_autoptr(GSocket) ipc_sock = g_socket_new_from_fd (RPMOSTREE_SHLIB_IPC_FD, error);
+  if (!ipc_sock)
+    return FALSE;
+  GOutputVector ov;
+  char buffer[1];
+  buffer[0] = 0xFF;
+  ov.buffer = buffer;
+  ov.size = G_N_ELEMENTS (buffer);
+  gssize r = g_socket_send_message (ipc_sock, NULL, &ov, 1,
+                                    (GSocketControlMessage **) &message,
+                                    1, 0, NULL, error);
+  if (r < 0)
+    return FALSE;
+  g_assert_cmpint (r, ==, 1);
+
+  return TRUE;
+}
+
+gboolean
+rpmostree_builtin_shlib_backend (int             argc,
+                                 char          **argv,
+                                 RpmOstreeCommandInvocation *invocation,
+                                 GCancellable   *cancellable,
+                                 GError        **error)
+{
+  if (argc < 2)
+    return glnx_throw (error, "missing required subcommand");
+
+  const char *arg = argv[1];
+
+  g_autoptr(GVariant) ret = NULL;
+
+  if (g_str_equal (arg, "get-basearch"))
+    {
+      g_autoptr(DnfContext) ctx = dnf_context_new ();
+      ret = g_variant_new_string (dnf_context_get_base_arch (ctx));
+    }
+  else if (g_str_equal (arg, "varsubst-basearch"))
+    {
+      const char *src = argv[2];
+      g_autoptr(DnfContext) ctx = dnf_context_new ();
+      g_autoptr(GHashTable) varsubsts = rpmostree_dnfcontext_get_varsubsts (ctx);
+      g_autofree char *rets = _rpmostree_varsubst_string (src, varsubsts, error);
+      if (rets == NULL)
+        return FALSE;
+      ret = g_variant_new_string (rets);
+    }
+  else
+    return glnx_throw (error, "unknown shlib-backend %s", arg);
+
+  glnx_fd_close int ret_memfd = memfd_create ("rpm-ostree-shlib-backend", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+  if (ret_memfd < 0)
+    return glnx_throw_errno_prefix (error, "memfd_create");
+  if (glnx_loop_write (ret_memfd, g_variant_get_data (ret), g_variant_get_size (ret)) < 0)
+    return glnx_throw_errno_prefix (error, "Failed to write to memfd");
+  const int seals = (F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_WRITE);
+  if (fcntl (ret_memfd, F_ADD_SEALS, seals) == -1)
+    return glnx_throw_errno_prefix (error, "fcntl(sealing)");
+
+  return send_memfd_result (glnx_steal_fd (&ret_memfd), error);
+}

src/app/rpmostree-builtins.h

@@ -52,6 +52,7 @@ BUILTINPROTO(override);
 BUILTINPROTO(kargs);
 BUILTINPROTO(reset);
 BUILTINPROTO(start_daemon);
+BUILTINPROTO(shlib_backend);
 BUILTINPROTO(coreos_rootfs);
 BUILTINPROTO(testutils);
 BUILTINPROTO(ex);

src/lib/rpmostree-shlib-ipc-private.h

@@ -0,0 +1,29 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2020 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+G_BEGIN_DECLS
+
+#define RPMOSTREE_SHLIB_IPC_FD 3
+
+GVariant *_rpmostree_shlib_ipc_send (const char *variant_type, char **args, GError **error);
+
+G_END_DECLS

src/lib/rpmostree.c

@@ -20,11 +20,15 @@
 
 #include "config.h"
 
+#include <gio/gio.h>
+#include <gio/gunixfdmessage.h>
+#include <gio/gunixsocketaddress.h>
 #include "string.h"
 
 #include "rpmostree.h"
 #include "rpmostree-core.h"
 #include "rpmostree-util.h"
+#include "rpmostree-shlib-ipc-private.h"
 
 /**
  * SECTION:librpmostree
@@ -34,6 +38,69 @@
  * These APIs access generic global state.
  */
 
+#define IPC_FD 3
+
+GVariant *
+_rpmostree_shlib_ipc_send (const char *variant_type, char **args, GError **error)
+{
+  g_autoptr(GSubprocessLauncher) launcher = g_subprocess_launcher_new (G_SUBPROCESS_FLAGS_STDOUT_SILENCE | G_SUBPROCESS_FLAGS_STDERR_PIPE);
+  int pair[2];
+  if (socketpair (AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, pair) < 0)
+    return (GVariant*)glnx_null_throw_errno_prefix (error, "couldn't create socket pair");
+  glnx_fd_close int my_sock_fd = glnx_steal_fd (&pair[0]);
+  g_subprocess_launcher_take_fd (launcher, pair[1], IPC_FD);
+
+  g_autoptr(GSocket) my_sock = g_socket_new_from_fd (my_sock_fd, error);
+  if (!my_sock)
+    return NULL;
+  my_sock_fd = -1; /* Ownership was transferred */
+  g_autoptr(GPtrArray) full_args = g_ptr_array_new ();
+  g_ptr_array_add (full_args, "rpm-ostree");
+  g_ptr_array_add (full_args, "shlib-backend");
+  for (char **it = args; it && *it; it++)
+    g_ptr_array_add (full_args, *it);
+  g_ptr_array_add (full_args, NULL);
+  g_autoptr(GSubprocess) proc = g_subprocess_launcher_spawnv (launcher, (const char*const*)full_args->pdata, error);
+  if (!proc)
+    return NULL;
+  
+  g_autofree char *stderr = NULL;
+  if (!g_subprocess_communicate_utf8 (proc, NULL, NULL, NULL, &stderr, error))
+    return NULL;
+
+  if (!g_subprocess_get_successful (proc))
+    return glnx_null_throw (error, "Failed to invoke rpm-ostree shlib-backend: %s", stderr);
+  int flags = 0;
+  int nm = 0;
+  GInputVector iv;
+  GUnixFDMessage **mv;
+  guint8 buffer[1024];
+  iv.buffer = buffer;
+  iv.size = 1;
+  gssize r = g_socket_receive_message (my_sock, NULL, &iv, 1,
+                                       (GSocketControlMessage ***) &mv,
+                                       &nm, &flags, NULL, error);
+  if (r < 0)
+    return NULL;
+  g_assert_cmpint (r, ==, 1);
+
+  if (nm != 1)
+    return glnx_null_throw (error, "Got %d control messages, expected 1", nm);
+  GUnixFDMessage *message = mv[0];
+  g_assert (G_IS_UNIX_FD_MESSAGE (message));
+  GUnixFDList *fdlist = g_unix_fd_message_get_fd_list (message);
+  const int nfds = g_unix_fd_list_get_length (fdlist);
+  if (nfds != 1)
+    return glnx_null_throw (error, "Got %d fds, expected 1", nfds);
+  const int *fds = g_unix_fd_list_peek_fds (fdlist, NULL);
+  const int result_memfd = fds[0];
+  g_assert_cmpint (result_memfd, !=, -1);
+  g_autoptr(GMappedFile) retmap = g_mapped_file_new_from_fd (result_memfd, FALSE, error);
+  if (!retmap)
+    return FALSE;
+  return g_variant_new_from_bytes ((GVariantType*) variant_type, g_mapped_file_get_bytes (retmap), FALSE);
+}
+
 /**
  * rpm_ostree_get_basearch:
  *
@@ -43,9 +110,11 @@
 char *
 rpm_ostree_get_basearch (void)
 {
-  g_autoptr(DnfContext) ctx = dnf_context_new ();
+  g_autoptr(GError) local_error = NULL;
-  /* Need to strdup since we unref the context */
+  char *args[] = { "get-basearch", NULL };
-  return g_strdup (dnf_context_get_base_arch (ctx));
+  g_autoptr(GVariant) ret = _rpmostree_shlib_ipc_send ("s", args, &local_error);
+  g_assert_no_error (local_error);
+  return g_variant_dup_string (ret, NULL);
 }
 
 /**
@@ -58,9 +127,11 @@ rpm_ostree_get_basearch (void)
 char *
 rpm_ostree_varsubst_basearch (const char *src, GError **error)
 {
-  g_autoptr(DnfContext) ctx = dnf_context_new ();
+  g_autoptr(GError) local_error = NULL;
-  g_autoptr(GHashTable) varsubsts = rpmostree_dnfcontext_get_varsubsts (ctx);
+  char *args[] = { "varsubst-basearch", (char*)src, NULL };
-  return _rpmostree_varsubst_string (src, varsubsts, error);
+  g_autoptr(GVariant) ret = _rpmostree_shlib_ipc_send ("s", args, &local_error);
+  g_assert_no_error (local_error);
+  return g_variant_dup_string (ret, NULL);
 }
 
 /**