Merge pull request #93 from cgwalters/passwd-deduplicate

Passwd deduplicate

src/rpmostree-passwd-util.c

@@ -668,69 +668,295 @@ rpmostree_check_groups (OstreeRepo      *repo,
                                         treedata, cancellable, error);
 }
 
+static FILE *
+gfopen (const char       *path,
+        const char       *mode,
+        GCancellable     *cancellable,
+        GError          **error)
+{
+  FILE *ret = NULL; 
+
+  ret = fopen (path, mode);
+  if (!ret)
+    {
+      _rpmostree_set_prefix_error_from_errno (error, errno, "fopen(%s): ", path);
+      return NULL;
+    }
+  return ret;
+}
+
+static gboolean
+gfflush (FILE         *f,
+         GCancellable *cancellable,
+         GError      **error)
+{
+  if (fflush (f) != 0)
+    {
+      _rpmostree_set_prefix_error_from_errno (error, errno, "fflush: ");
+      return FALSE;
+    }
+  return TRUE;
+}
+
+/*
+ * This function is taking the /etc/passwd generated in the install
+ * root, and splitting it into two streams: a new /etc/passwd that
+ * just contains the root entry, and /usr/lib/passwd which contains
+ * everything else.
+ *
+ * The implementation is kind of horrible because I wanted to avoid
+ * duplicating the user/group code.
+ */
+gboolean
+rpmostree_passwd_migrate_except_root (GFile         *rootfs,
+                                      RpmOstreePasswdMigrateKind    kind,
+                                      GHashTable    *preserve,
+                                      GCancellable  *cancellable,
+                                      GError       **error)
+{
+  gboolean ret = FALSE;
+  const char *name = kind == RPM_OSTREE_PASSWD_MIGRATE_PASSWD ? "passwd" : "group";
+  gs_free char *src_path = g_strconcat (gs_file_get_path_cached (rootfs), "/etc/", name, NULL);
+  gs_free char *etctmp_path = g_strconcat (gs_file_get_path_cached (rootfs), "/etc/", name, ".tmp", NULL);
+  gs_free char *usrdest_path = g_strconcat (gs_file_get_path_cached (rootfs), "/usr/lib/", name, NULL);
+  FILE *src_stream = NULL;
+  FILE *etcdest_stream = NULL;
+  FILE *usrdest_stream = NULL;
+
+  src_stream = gfopen (src_path, "r", cancellable, error);
+  if (!src_stream)
+    goto out;
+
+  etcdest_stream = gfopen (etctmp_path, "w", cancellable, error);
+  if (!etcdest_stream)
+    goto out;
+
+  usrdest_stream = gfopen (usrdest_path, "a", cancellable, error);
+  if (!usrdest_stream)
+    goto out;
+
+  errno = 0;
+  while (TRUE)
+    {
+      struct passwd *pw = NULL;
+      struct group *gr = NULL;
+      FILE *deststream;
+      int r;
+      guint32 id;
+      const char *name;
+      
+      if (kind == RPM_OSTREE_PASSWD_MIGRATE_PASSWD)
+        pw = fgetpwent (src_stream);
+      else
+        gr = fgetgrent (src_stream);
+
+      if (!(pw || gr))
+        {
+          if (errno != 0 && errno != ENOENT)
+            {
+              _rpmostree_set_prefix_error_from_errno (error, errno, "fgetpwent: ");
+              goto out;
+            }
+          else
+            break;
+        }
+
+
+      if (pw)
+        {
+          id = pw->pw_uid;
+          name = pw->pw_name;
+        }
+      else
+        {
+          id = gr->gr_gid;
+          name = gr->gr_name;
+        }
+
+      if (id == 0)
+        deststream = etcdest_stream;
+      else
+        deststream = usrdest_stream;
+
+      if (pw)
+        r = putpwent (pw, deststream);
+      else
+        r = putgrent (gr, deststream);
+
+      /* If it's marked in the preserve group, we need to write to
+       * *both* /etc and /usr/lib in order to preserve semantics for
+       * upgraded systems from before we supported the preserve
+       * concept.
+       */
+      if (preserve && g_hash_table_contains (preserve, name))
+        {
+          /* We should never be trying to preserve the root entry, it
+           * should always be only in /etc.
+           */
+          g_assert (deststream == usrdest_stream);
+          if (pw)
+            r = putpwent (pw, etcdest_stream);
+          else
+            r = putgrent (gr, etcdest_stream);
+        }
+
+      if (r == -1)
+        {
+          _rpmostree_set_prefix_error_from_errno (error, errno, "putpwent: ");
+          goto out;
+        }
+    }
+
+  if (!gfflush (etcdest_stream, cancellable, error))
+    goto out;
+  if (!gfflush (usrdest_stream, cancellable, error))
+    goto out;
+
+  if (rename (etctmp_path, src_path) != 0)
+    {
+      _rpmostree_set_prefix_error_from_errno (error, errno, "rename(%s, %s): ",
+                                              etctmp_path, src_path);
+      goto out;
+    }
+
+  ret = TRUE;
+ out:
+  if (src_stream) (void) fclose (src_stream);
+  if (etcdest_stream) (void) fclose (etcdest_stream);
+  if (usrdest_stream) (void) fclose (usrdest_stream);
+  return ret;
+}
+
 static gboolean
 concat_passwd_file (GFile           *yumroot,
                     GFile           *previous_commit,
-                    const char      *filename,
+                    RpmOstreePasswdMigrateKind kind,
                     GCancellable    *cancellable,
                     GError         **error)
 {
   gboolean ret = FALSE;
+  const char *filename = kind == RPM_OSTREE_PASSWD_MIGRATE_PASSWD ? "passwd" : "group";
   gs_free char *etc_subpath = g_strconcat ("etc/", filename, NULL);
   gs_free char *usretc_subpath = g_strconcat ("usr/etc/", filename, NULL);
   gs_free char *usrlib_subpath = g_strconcat ("usr/lib/", filename, NULL);
   gs_unref_object GFile *yumroot_etc = g_file_resolve_relative_path (yumroot, "etc");
-  gs_unref_object GFile *yumroot_dest = g_file_resolve_relative_path (yumroot, etc_subpath);
+  gs_unref_object GFile *orig_etc_content =
-  gs_unref_object GFile *orig_etc_content = g_file_resolve_relative_path (previous_commit, usretc_subpath);
+    g_file_resolve_relative_path (previous_commit, usretc_subpath);
-  gs_unref_object GFile *orig_usrlib_content = g_file_resolve_relative_path (previous_commit, usrlib_subpath);
+  gs_unref_object GFile *orig_usrlib_content =
+    g_file_resolve_relative_path (previous_commit, usrlib_subpath);
   gs_unref_object GFileOutputStream *out = NULL;
+  gs_unref_hashtable GHashTable *seen_names =
+    g_hash_table_new_full (g_str_hash, g_str_equal, NULL, g_free);
+  gs_free char *contents = NULL;
+  GFile *sources[] = { orig_etc_content, orig_usrlib_content };
+  guint i;
+  gsize len;
   gboolean have_etc, have_usr;
+  FILE *src_stream = NULL;
+  FILE *dest_stream = NULL;
 
+  /* Create /etc in the target root; FIXME - should ensure we're using
+   * the right permissions from the filesystem RPM.  Doing this right
+   * is really hard because filesystem depends on setup which installs
+   * the files...
+   */
   if (!gs_file_ensure_directory (yumroot_etc, TRUE, cancellable, error))
     goto out;
 
   have_etc = g_file_query_exists (orig_etc_content, NULL);
   have_usr = g_file_query_exists (orig_usrlib_content, NULL);
 
+  /* This could actually happen after we transition to
+   * systemd-sysusers; we won't have a need for preallocated user data
+   * in the tree.
+   */
   if (!(have_etc || have_usr))
     {
       ret = TRUE;
       goto out;
     }
 
-  out = g_file_replace (yumroot_dest, NULL, FALSE, G_FILE_CREATE_REPLACE_DESTINATION,
+  { gs_free char *target_etc =
-                        cancellable, error);
+      g_build_filename (gs_file_get_path_cached (yumroot), etc_subpath, NULL);
-  if (!out)
+    dest_stream = gfopen (target_etc, "w", cancellable, error);
-    goto out;
+    if (!dest_stream)
+      goto out;
+  }
 
-  if (have_etc)
+  for (i = 0; i < G_N_ELEMENTS (sources); i++)
     {
-      gs_unref_object GInputStream *src =
+      GFile *source = sources[i];
-        (GInputStream*)g_file_read (orig_etc_content, cancellable, error);
+
-      if (g_output_stream_splice ((GOutputStream*)out, src,
+      /* We read the file into memory using Gio (which talks
-                                  G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
+       * to libostree), then memopen it, which works with libc.
-                                  cancellable, error ) < 0)
+       */
+      if (!g_file_load_contents (source, cancellable,
+                                 &contents, &len, NULL, error))
         goto out;
+      
+      if (src_stream) (void) fclose (src_stream);
+      src_stream = fmemopen (contents, len, "r");
+      if (!src_stream)
+        {
+          gs_set_error_from_errno (error, errno);
+          goto out;
+        }
+      
+      errno = 0;
+      while (TRUE)
+        {
+          struct passwd *pw = NULL;
+          struct group *gr = NULL;
+          int r;
+          const char *name;
+      
+          if (kind == RPM_OSTREE_PASSWD_MIGRATE_PASSWD)
+            pw = fgetpwent (src_stream);
+          else
+            gr = fgetgrent (src_stream);
+
+          if (!(pw || gr))
+            {
+              if (errno != 0 && errno != ENOENT)
+                {
+                  _rpmostree_set_prefix_error_from_errno (error, errno, "fgetpwent: ");
+                  goto out;
+                }
+              else
+                break;
+            }
+
+          if (pw)
+            name = pw->pw_name;
+          else
+            name = gr->gr_name;
+
+          /* Deduplicate */
+          if (g_hash_table_lookup (seen_names, name))
+            continue;
+          g_hash_table_add (seen_names, g_strdup (name));
+
+          if (pw)
+            r = putpwent (pw, dest_stream);
+          else
+            r = putgrent (gr, dest_stream);
+          if (r == -1)
+            {
+              _rpmostree_set_prefix_error_from_errno (error, errno, "putpwent: ");
+              goto out;
+            }
+        }
     }
 
-  if (have_usr)
+  if (!gfflush (dest_stream, cancellable, error))
-    {
-      gs_unref_object GInputStream *src =
-        (GInputStream*)g_file_read (orig_usrlib_content, cancellable, error);
-      if (g_output_stream_splice ((GOutputStream*)out, src,
-                                  G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
-                                  cancellable, error ) < 0)
-        goto out;
-    }
-
-  if (!g_output_stream_flush ((GOutputStream*)out, cancellable, error))
     goto out;
 
   ret = TRUE;
  out:
+  if (src_stream) (void) fclose (src_stream);
+  if (dest_stream) (void) fclose (dest_stream);
   return ret;
 }
-                    
 
 gboolean
 rpmostree_generate_passwd_from_previous (OstreeRepo      *repo,
@@ -740,14 +966,15 @@ rpmostree_generate_passwd_from_previous (OstreeRepo      *repo,
                                          GError         **error)
 {
   gboolean ret = FALSE;
-  gs_unref_object GFile *yumroot_etc_group = g_file_resolve_relative_path (yumroot, "etc/group");
   gs_unref_object GFile *out = NULL;
 
-  if (!concat_passwd_file (yumroot, previous_root, "passwd",
+  if (!concat_passwd_file (yumroot, previous_root,
+                           RPM_OSTREE_PASSWD_MIGRATE_PASSWD,
                            cancellable, error))
     goto out;
 
-  if (!concat_passwd_file (yumroot, previous_root, "group",
+  if (!concat_passwd_file (yumroot, previous_root,
+                           RPM_OSTREE_PASSWD_MIGRATE_GROUP,
                            cancellable, error))
     goto out;
 

src/rpmostree-passwd-util.h

@@ -40,6 +40,18 @@ rpmostree_check_groups (OstreeRepo      *repo,
                         GCancellable    *cancellable,
                         GError         **error);
 
+typedef enum {
+  RPM_OSTREE_PASSWD_MIGRATE_PASSWD,
+  RPM_OSTREE_PASSWD_MIGRATE_GROUP
+} RpmOstreePasswdMigrateKind;
+
+gboolean
+rpmostree_passwd_migrate_except_root (GFile         *rootfs,
+                                      RpmOstreePasswdMigrateKind    kind,
+                                      GHashTable    *preserve,
+                                      GCancellable  *cancellable,
+                                      GError       **error);
+
 gboolean
 rpmostree_generate_passwd_from_previous (OstreeRepo      *repo,
                                          GFile           *yumroot,

src/rpmostree-postprocess.c

@@ -34,6 +34,7 @@
 #include <stdlib.h>
 
 #include "rpmostree-postprocess.h"
+#include "rpmostree-passwd-util.h"
 #include "rpmostree-libcontainer.h"
 #include "rpmostree-cleanup.h"
 #include "rpmostree-json-parsing.h"
@@ -593,170 +594,6 @@ workaround_selinux_cross_labeling (GFile         *rootfs,
   return ret;
 }
 
-static FILE *
-gfopen (const char       *path,
-        const char       *mode,
-        GCancellable     *cancellable,
-        GError          **error)
-{
-  FILE *ret = NULL; 
-
-  ret = fopen (path, mode);
-  if (!ret)
-    {
-      _rpmostree_set_prefix_error_from_errno (error, errno, "fopen(%s): ", path);
-      return NULL;
-    }
-  return ret;
-}
-
-static gboolean
-gfflush (FILE         *f,
-         GCancellable *cancellable,
-         GError      **error)
-{
-  if (fflush (f) != 0)
-    {
-      _rpmostree_set_prefix_error_from_errno (error, errno, "fflush: ");
-      return FALSE;
-    }
-  return TRUE;
-}
-
-typedef enum {
-  MIGRATE_PASSWD,
-  MIGRATE_GROUP
-} MigrateKind;
-
-/*
- * This function is taking the /etc/passwd generated in the install
- * root, and splitting it into two streams: a new /etc/passwd that
- * just contains the root entry, and /usr/lib/passwd which contains
- * everything else.
- *
- * The implementation is kind of horrible because I wanted to avoid
- * duplicating the user/group code.
- */
-static gboolean
-migrate_passwd_file_except_root (GFile         *rootfs,
-                                 MigrateKind    kind,
-                                 GHashTable    *preserve,
-                                 GCancellable  *cancellable,
-                                 GError       **error)
-{
-  gboolean ret = FALSE;
-  const char *name = kind == MIGRATE_PASSWD ? "passwd" : "group";
-  gs_free char *src_path = g_strconcat (gs_file_get_path_cached (rootfs), "/etc/", name, NULL);
-  gs_free char *etctmp_path = g_strconcat (gs_file_get_path_cached (rootfs), "/etc/", name, ".tmp", NULL);
-  gs_free char *usrdest_path = g_strconcat (gs_file_get_path_cached (rootfs), "/usr/lib/", name, NULL);
-  FILE *src_stream = NULL;
-  FILE *etcdest_stream = NULL;
-  FILE *usrdest_stream = NULL;
-
-  src_stream = gfopen (src_path, "r", cancellable, error);
-  if (!src_stream)
-    goto out;
-
-  etcdest_stream = gfopen (etctmp_path, "w", cancellable, error);
-  if (!etcdest_stream)
-    goto out;
-
-  usrdest_stream = gfopen (usrdest_path, "a", cancellable, error);
-  if (!usrdest_stream)
-    goto out;
-
-  errno = 0;
-  while (TRUE)
-    {
-      struct passwd *pw = NULL;
-      struct group *gr = NULL;
-      FILE *deststream;
-      int r;
-      guint32 id;
-      const char *name;
-      
-      if (kind == MIGRATE_PASSWD)
-        pw = fgetpwent (src_stream);
-      else
-        gr = fgetgrent (src_stream);
-
-      if (!(pw || gr))
-        {
-          if (errno != 0 && errno != ENOENT)
-            {
-              _rpmostree_set_prefix_error_from_errno (error, errno, "fgetpwent: ");
-              goto out;
-            }
-          else
-            break;
-        }
-
-
-      if (pw)
-        {
-          id = pw->pw_uid;
-          name = pw->pw_name;
-        }
-      else
-        {
-          id = gr->gr_gid;
-          name = gr->gr_name;
-        }
-
-      if (id == 0)
-        deststream = etcdest_stream;
-      else
-        deststream = usrdest_stream;
-
-      if (pw)
-        r = putpwent (pw, deststream);
-      else
-        r = putgrent (gr, deststream);
-
-      /* If it's marked in the preserve group, we need to write to
-       * *both* /etc and /usr/lib in order to preserve semantics for
-       * upgraded systems from before we supported the preserve
-       * concept.
-       */
-      if (preserve && g_hash_table_contains (preserve, name))
-        {
-          /* We should never be trying to preserve the root entry, it
-           * should always be only in /etc.
-           */
-          g_assert (deststream == usrdest_stream);
-          if (pw)
-            r = putpwent (pw, etcdest_stream);
-          else
-            r = putgrent (gr, etcdest_stream);
-        }
-
-      if (r == -1)
-        {
-          _rpmostree_set_prefix_error_from_errno (error, errno, "putpwent: ");
-          goto out;
-        }
-    }
-
-  if (!gfflush (etcdest_stream, cancellable, error))
-    goto out;
-  if (!gfflush (usrdest_stream, cancellable, error))
-    goto out;
-
-  if (rename (etctmp_path, src_path) != 0)
-    {
-      _rpmostree_set_prefix_error_from_errno (error, errno, "rename(%s, %s): ",
-                                              etctmp_path, src_path);
-      goto out;
-    }
-
-  ret = TRUE;
- out:
-  if (src_stream) (void) fclose (src_stream);
-  if (etcdest_stream) (void) fclose (etcdest_stream);
-  if (usrdest_stream) (void) fclose (usrdest_stream);
-  return ret;
-}
-
 static gboolean
 replace_nsswitch (GFile         *target_usretc,
                   GCancellable  *cancellable,
@@ -908,8 +745,8 @@ create_rootfs_from_yumroot_content (GFile         *targetroot,
     goto out;
 
   g_print ("Migrating /etc/passwd to /usr/lib/\n");
-  if (!migrate_passwd_file_except_root (yumroot, MIGRATE_PASSWD, NULL,
+  if (!rpmostree_passwd_migrate_except_root (yumroot, RPM_OSTREE_PASSWD_MIGRATE_PASSWD, NULL,
-                                        cancellable, error))
+                                             cancellable, error))
     goto out;
 
   if (json_object_has_member (treefile, "etc-group-members"))
@@ -919,8 +756,9 @@ create_rootfs_from_yumroot_content (GFile         *targetroot,
     }
       
   g_print ("Migrating /etc/group to /usr/lib/\n");
-  if (!migrate_passwd_file_except_root (yumroot, MIGRATE_GROUP, preserve_groups_set,
+  if (!rpmostree_passwd_migrate_except_root (yumroot, RPM_OSTREE_PASSWD_MIGRATE_GROUP,
-                                        cancellable, error))
+                                             preserve_groups_set,
+                                             cancellable, error))
     goto out;
 
   /* NSS configuration to look at the new files */