compose+rust: Parse includes via Rust too

This follows up to https://github.com/projectatomic/rpm-ostree/pull/1576
AKA commit 2e567840ca - we now process
treefile inheritance in Rust code.  Previously for elements which
reference external files (`postprocess-script` and `add-files`)
we'd hardcoded things to only look in the first context dir.

Now we open file descriptors in the Rust side for these "externals"
as we're parsing, and load them C side.  Hence we'll correctly handle
a `postprocess-script` from an included config.

Other advantages are that the include handling was ugly un-typesafe C code
with no unit tests, now it's memory safe Rust with unit tests.

The downside here is I ended up spelling out the list of fields
again - there's probably a way to unify this via macros but
for now I think this is OK.

Closes: #1574
Approved by: jlebon

Makefile-lib.am

@@ -34,7 +34,7 @@ librpmostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/libglnx -I$(srcdir)/src/libp
 	-fvisibility=hidden '-D_RPMOSTREE_EXTERN=__attribute((visibility("default"))) extern' \
 	$(PKGDEP_RPMOSTREE_CFLAGS)
 librpmostree_1_la_LDFLAGS = $(AM_LDFLAGS) -version-number 1:0:0 -Bsymbolic-functions
-librpmostree_1_la_LIBADD =  $(PKGDEP_RPMOSTREE_LIBS) librpmostreepriv.la
+librpmostree_1_la_LIBADD =  $(PKGDEP_RPMOSTREE_LIBS) librpmostreepriv.la $(librpmostree_rust_path)
 
 # bundled libdnf
 INTROSPECTION_SCANNER_ENV = env LD_LIBRARY_PATH=$(top_builddir)/libdnf-build/libdnf

rust/src/lib.rs

@@ -32,8 +32,9 @@ extern crate serde_json;
 extern crate serde_yaml;
 
 use std::ffi::{CStr, OsStr};
+use std::io::Seek;
 use std::os::unix::ffi::OsStrExt;
-use std::os::unix::io::{FromRawFd, IntoRawFd};
+use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};
 use std::{io, ptr};
 
 mod glibutils;
@@ -94,6 +95,41 @@ pub extern "C" fn ror_treefile_new(
     )
 }
 
+#[no_mangle]
+pub extern "C" fn ror_treefile_get_dfd(tf: *mut Treefile) -> libc::c_int {
+    assert!(!tf.is_null());
+    let tf = unsafe { &mut *tf };
+    tf.primary_dfd.as_raw_fd()
+}
+
+#[no_mangle]
+pub extern "C" fn ror_treefile_get_postprocess_script_fd(tf: *mut Treefile) -> libc::c_int {
+    assert!(!tf.is_null());
+    let tf = unsafe { &mut *tf };
+    if let Some(ref mut fd) = tf.externals.postprocess_script.as_ref() {
+        // We always seek to the start
+        fd.seek(io::SeekFrom::Start(0)).unwrap();
+        fd.as_raw_fd()
+    } else {
+        -1
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn ror_treefile_get_add_file_fd(
+    tf: *mut Treefile,
+    filename: *const libc::c_char,
+) -> libc::c_int {
+    assert!(!tf.is_null());
+    let tf = unsafe { &mut *tf };
+    let filename = OsStr::from_bytes(bytes_from_nonnull(filename));
+    let filename = filename.to_string_lossy().into_owned();
+    let mut fd = tf.externals.add_files.get(&filename).expect("add-file");
+    // We always seek to the start
+    fd.seek(io::SeekFrom::Start(0)).unwrap();
+    fd.as_raw_fd()
+}
+
 #[no_mangle]
 pub extern "C" fn ror_treefile_to_json(
     tf: *mut Treefile,

rust/src/treefile.rs

@@ -26,15 +26,31 @@ use serde_json;
 use serde_yaml;
 use std::io::prelude::*;
 use std::path::Path;
-use std::{fs, io};
+use std::{collections, fs, io};
 use tempfile;
 
 const ARCH_X86_64: &'static str = "x86_64";
+const INCLUDE_MAXDEPTH: u32 = 50;
+
+/// This struct holds file descriptors for any external files/data referenced by
+/// a TreeComposeConfig.
+pub struct TreefileExternals {
+    pub postprocess_script: Option<fs::File>,
+    pub add_files: collections::HashMap<String, fs::File>,
+}
 
 pub struct Treefile {
     pub workdir: openat::Dir,
+    pub primary_dfd: openat::Dir,
     pub parsed: TreeComposeConfig,
     pub rojig_spec: Option<CUtf8Buf>,
+    pub externals: TreefileExternals,
+}
+
+// We only use this while parsing
+struct ConfigAndExternals {
+    config: TreeComposeConfig,
+    externals: TreefileExternals,
 }
 
 enum InputFormat {
@@ -43,6 +59,7 @@ enum InputFormat {
 }
 
 /// Parse a YAML treefile definition using architecture `arch`.
+/// This does not open the externals.
 fn treefile_parse_stream<R: io::Read>(
     fmt: InputFormat,
     input: &mut R,
@@ -100,32 +117,175 @@ fn treefile_parse_stream<R: io::Read>(
     Ok(treefile)
 }
 
+/// Given a treefile filename and an architecture, parse it and also
+/// open its external files.
+fn treefile_parse<P: AsRef<Path>>(
+    filename: P,
+    arch: Option<&str>,
+) -> io::Result<ConfigAndExternals> {
+    let filename = filename.as_ref();
+    let mut f = io::BufReader::new(fs::File::open(filename)?);
+    let basename = filename
+        .file_name()
+        .map(|s| s.to_string_lossy())
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Expected a filename"))?;
+    let fmt = if basename.ends_with(".yaml") || basename.ends_with(".yml") {
+        InputFormat::YAML
+    } else {
+        InputFormat::JSON
+    };
+    let tf = treefile_parse_stream(fmt, &mut f, arch).map_err(|e| {
+        io::Error::new(
+            io::ErrorKind::InvalidInput,
+            format!("Parsing {}: {}", filename.to_string_lossy(), e.to_string()),
+        )
+    })?;
+    let postprocess_script = if let Some(ref postprocess) = tf.postprocess_script.as_ref() {
+        Some(fs::File::open(filename.with_file_name(postprocess))?)
+    } else {
+        None
+    };
+    let mut add_files: collections::HashMap<String, fs::File> = collections::HashMap::new();
+    if let Some(ref add_file_names) = tf.add_files.as_ref() {
+        for (name, _) in add_file_names.iter() {
+            add_files.insert(name.clone(), fs::File::open(filename.with_file_name(name))?);
+        }
+    }
+    Ok(ConfigAndExternals {
+        config: tf,
+        externals: TreefileExternals {
+            postprocess_script,
+            add_files,
+        },
+    })
+}
+
+/// Merge a "basic" or non-array field. The semantics originally defined for
+/// these are that first-one wins.
+fn merge_basic_field<T>(dest: &mut Option<T>, src: &mut Option<T>) {
+    if dest.is_some() {
+        return;
+    }
+    *dest = src.take()
+}
+
+/// Merge a vector field by appending. This semantic was originally designed for
+/// the `packages` key.
+fn merge_vec_field<T>(dest: &mut Option<Vec<T>>, src: &mut Option<Vec<T>>) {
+    if let Some(mut srcv) = src.take() {
+        if let Some(ref mut destv) = dest {
+            srcv.append(destv);
+        }
+        *dest = Some(srcv);
+    }
+}
+
+/// Given two configs, merge them. Ideally we'd do some macro magic and avoid
+/// listing all of the fields again.
+fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
+    merge_basic_field(&mut dest.treeref, &mut src.treeref);
+    merge_basic_field(&mut dest.rojig, &mut src.rojig);
+    merge_vec_field(&mut dest.repos, &mut src.repos);
+    merge_basic_field(&mut dest.selinux, &mut src.selinux);
+    merge_basic_field(&mut dest.gpg_key, &mut src.gpg_key);
+    merge_basic_field(&mut dest.include, &mut src.include);
+    merge_vec_field(&mut dest.packages, &mut src.packages);
+    merge_basic_field(&mut dest.recommends, &mut src.recommends);
+    merge_basic_field(&mut dest.documentation, &mut src.documentation);
+    merge_vec_field(&mut dest.install_langs, &mut src.install_langs);
+    merge_vec_field(&mut dest.initramfs_args, &mut src.initramfs_args);
+    merge_basic_field(&mut dest.boot_location, &mut src.boot_location);
+    merge_basic_field(&mut dest.tmp_is_dir, &mut src.tmp_is_dir);
+    merge_basic_field(&mut dest.default_target, &mut src.default_target);
+    merge_vec_field(&mut dest.units, &mut src.units);
+    merge_basic_field(&mut dest.machineid_compat, &mut src.machineid_compat);
+    merge_basic_field(&mut dest.releasever, &mut src.releasever);
+    merge_basic_field(
+        &mut dest.automatic_version_prefix,
+        &mut src.automatic_version_prefix,
+    );
+    merge_basic_field(&mut dest.mutate_os_release, &mut src.mutate_os_release);
+    merge_basic_field(&mut dest.etc_group_members, &mut src.etc_group_members);
+    merge_basic_field(&mut dest.preserve_passwd, &mut src.preserve_passwd);
+    merge_basic_field(&mut dest.check_passwd, &mut src.check_passwd);
+    merge_basic_field(&mut dest.check_groups, &mut src.check_groups);
+    merge_basic_field(
+        &mut dest.ignore_removed_users,
+        &mut src.ignore_removed_users,
+    );
+    merge_basic_field(
+        &mut dest.ignore_removed_groups,
+        &mut src.ignore_removed_groups,
+    );
+    merge_basic_field(&mut dest.postprocess_script, &mut src.postprocess_script);
+    merge_vec_field(&mut dest.postprocess, &mut src.postprocess);
+    merge_vec_field(&mut dest.add_files, &mut src.add_files);
+    merge_vec_field(&mut dest.remove_files, &mut src.remove_files);
+    merge_vec_field(
+        &mut dest.remove_from_packages,
+        &mut src.remove_from_packages,
+    );
+}
+
+/// Merge the treefile externals. There are currently only two keys that
+/// reference external files.
+fn treefile_merge_externals(dest: &mut TreefileExternals, src: &mut TreefileExternals) {
+    // This one, being a basic-valued field, has first-wins semantics.
+    if dest.postprocess_script.is_none() {
+        dest.postprocess_script = src.postprocess_script.take();
+    }
+
+    // add-files is an array and hence has append semantics.
+    for (k, v) in src.add_files.drain() {
+        dest.add_files.insert(k, v);
+    }
+}
+
+/// Recursively parse a treefile, merging along the way.
+fn treefile_parse_recurse<P: AsRef<Path>>(
+    filename: P,
+    arch: Option<&str>,
+    depth: u32,
+) -> io::Result<ConfigAndExternals> {
+    let filename = filename.as_ref();
+    let mut parsed = treefile_parse(filename, arch)?;
+    let include_path = parsed.config.include.take();
+    if let &Some(ref include_path) = &include_path {
+        if depth == INCLUDE_MAXDEPTH {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                format!("Reached maximum include depth {}", INCLUDE_MAXDEPTH),
+            ));
+        }
+        let parent = filename.parent().unwrap();
+        let include_path = parent.join(include_path);
+        let mut included = treefile_parse_recurse(include_path, arch, depth + 1)?;
+        treefile_merge(&mut parsed.config, &mut included.config);
+        treefile_merge_externals(&mut parsed.externals, &mut included.externals);
+    }
+    Ok(parsed)
+}
+
 impl Treefile {
+    /// The main treefile creation entrypoint.
     pub fn new_boxed(
         filename: &Path,
         arch: Option<&str>,
         workdir: openat::Dir,
     ) -> io::Result<Box<Treefile>> {
-        let mut f = io::BufReader::new(fs::File::open(filename)?);
-        let basename = filename
-            .file_name()
-            .map(|s| s.to_string_lossy())
-            .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Expected a filename"))?;
-        let fmt = if basename.ends_with(".yaml") || basename.ends_with(".yml") {
-            InputFormat::YAML
-        } else {
-            InputFormat::JSON
-        };
-        let parsed = treefile_parse_stream(fmt, &mut f, arch)?;
-        let rojig_spec = if let &Some(ref rojig) = &parsed.rojig {
+        let parsed = treefile_parse_recurse(filename, arch, 0)?;
+        let dfd = openat::Dir::open(filename.parent().unwrap())?;
+        let rojig_spec = if let &Some(ref rojig) = &parsed.config.rojig {
             Some(Treefile::write_rojig_spec(&workdir, rojig)?)
         } else {
             None
         };
         Ok(Box::new(Treefile {
-            parsed: parsed,
+            primary_dfd: dfd,
+            parsed: parsed.config,
             workdir: workdir,
             rojig_spec: rojig_spec,
+            externals: parsed.externals,
         }))
     }
 
@@ -139,6 +299,7 @@ impl Treefile {
         Ok(tmpf)
     }
 
+    /// Generate a rojig spec file.
     fn write_rojig_spec<'a, 'b>(workdir: &'a openat::Dir, r: &'b Rojig) -> io::Result<CUtf8Buf> {
         let description = r
             .description
@@ -186,6 +347,8 @@ for x in *; do mv ${{x}} %{{buildroot}}%{{_prefix}}/lib/ostree-jigdo/%{{name}};
     }
 }
 
+/// For increased readability in YAML/JSON, we support whitespace in individual
+/// array elements.
 fn whitespace_split_packages(pkgs: &[String]) -> Vec<String> {
     pkgs.iter()
         .flat_map(|pkg| pkg.split_whitespace().map(String::from))
@@ -390,11 +553,6 @@ packages-x86_64:
  - grub2 grub2-tools
 packages-s390x:
  - zipl
-add-files:
- - - foo
-   - /usr/bin/foo
- - - baz
-   - /usr/bin/blah
 "###;
 
     // This one has "comments" (hence unknown keys)
@@ -416,7 +574,29 @@ add-files:
             treefile_parse_stream(InputFormat::YAML, &mut input, Some(ARCH_X86_64)).unwrap();
         assert!(treefile.treeref.unwrap() == "exampleos/x86_64/blah");
         assert!(treefile.packages.unwrap().len() == 5);
+    }
+
+    #[test]
+    fn basic_valid_add_remove_files() {
+        let mut buf = VALID_PRELUDE.to_string();
+        buf.push_str(
+            r###"
+add-files:
+  - - foo
+    - /usr/bin/foo
+  - - baz
+    - /usr/bin/blah
+remove-files:
+ - foo
+ - bar
+"###,
+        );
+        let buf = buf.as_bytes();
+        let mut input = io::BufReader::new(buf);
+        let treefile =
+            treefile_parse_stream(InputFormat::YAML, &mut input, Some(ARCH_X86_64)).unwrap();
         assert!(treefile.add_files.unwrap().len() == 2);
+        assert!(treefile.remove_files.unwrap().len() == 2);
     }
 
     #[test]
@@ -496,17 +676,17 @@ add-files:
         assert!(tf.parsed.machineid_compat.is_none());
     }
 
-    #[test]
-    fn test_treefile_new_rojig() {
-        let mut buf = VALID_PRELUDE.to_string();
-        buf.push_str(
-            r###"
+    const ROJIG_YAML: &'static str = r###"
 rojig:
   name: "exampleos"
   license: "MIT"
   summary: "ExampleOS rojig base image"
-"###,
-        );
+"###;
+
+    #[test]
+    fn test_treefile_new_rojig() {
+        let mut buf = VALID_PRELUDE.to_string();
+        buf.push_str(ROJIG_YAML);
         let t = TreefileTest::new(buf.as_str(), None).unwrap();
         let tf = &t.tf;
         let rojig = tf.parsed.rojig.as_ref().unwrap();
@@ -516,4 +696,25 @@ rojig:
         assert!(rojig_spec.file_name().unwrap() == "exampleos.spec");
     }
 
+    #[test]
+    fn test_treefile_merge() {
+        let arch = Some(ARCH_X86_64);
+        let mut base_input = io::BufReader::new(VALID_PRELUDE.as_bytes());
+        let mut base = treefile_parse_stream(InputFormat::YAML, &mut base_input, arch).unwrap();
+        let mut mid_input = io::BufReader::new(
+            r###"
+packages:
+  - some layered packages
+"###.as_bytes(),
+        );
+        let mut mid = treefile_parse_stream(InputFormat::YAML, &mut mid_input, arch).unwrap();
+        let mut top_input = io::BufReader::new(ROJIG_YAML.as_bytes());
+        let mut top = treefile_parse_stream(InputFormat::YAML, &mut top_input, arch).unwrap();
+        treefile_merge(&mut mid, &mut base);
+        treefile_merge(&mut top, &mut mid);
+        let tf = &top;
+        assert!(tf.packages.as_ref().unwrap().len() == 8);
+        let rojig = tf.rojig.as_ref().unwrap();
+        assert!(rojig.name == "exampleos");
+    }
 }

src/app/rpmostree-compose-builtin-tree.c

@@ -105,8 +105,6 @@ static GOptionEntry commit_option_entries[] = {
 };
 
 typedef struct {
-  GPtrArray *treefile_context_dirs;
-
   RpmOstreeContext *corectx;
   GFile *treefile_path;
   GHashTable *metadata;
@@ -133,7 +131,6 @@ typedef struct {
 static void
 rpm_ostree_tree_compose_context_free (RpmOstreeTreeComposeContext *ctx)
 {
-  g_clear_pointer (&ctx->treefile_context_dirs, (GDestroyNotify)g_ptr_array_unref);
   g_clear_object (&ctx->corectx);
   g_clear_object (&ctx->treefile_path);
   g_clear_pointer (&ctx->metadata, g_hash_table_unref);
@@ -263,8 +260,10 @@ install_packages_in_root (RpmOstreeTreeComposeContext  *self,
     rpmlogSetFile(NULL);
   }
 
-  GFile *contextdir = self->treefile_context_dirs->pdata[0];
-  dnf_context_set_repo_dir (dnfctx, gs_file_get_path_cached (contextdir));
+  { int tf_dfd = ror_treefile_get_dfd (self->treefile_rs);
+    g_autofree char *abs_tf_path = glnx_fdrel_abspath (tf_dfd, ".");
+    dnf_context_set_repo_dir (dnfctx, abs_tf_path);
+  }
 
   /* By default, retain packages in addition to metadata with --cachedir, unless
    * we're doing unified core, in which case the pkgcache repo is the cache.  But
@@ -374,7 +373,7 @@ install_packages_in_root (RpmOstreeTreeComposeContext  *self,
   g_autofree char *ret_new_inputhash = NULL;
   if (!rpmostree_composeutil_checksum (self->serialized_treefile,
                                        dnf_context_get_goal (dnfctx),
-                                       contextdir, add_files,
+                                       self->treefile_rs, add_files,
                                        &ret_new_inputhash, error))
     return FALSE;
 
@@ -403,7 +402,8 @@ install_packages_in_root (RpmOstreeTreeComposeContext  *self,
   if (opt_dry_run)
     return TRUE; /* NB: early return */
 
-  if (!rpmostree_composeutil_sanity_checks (treedata, self->treefile_context_dirs->pdata[0],
+  if (!rpmostree_composeutil_sanity_checks (self->treefile_rs,
+                                            self->treefile,
                                             cancellable, error))
     return FALSE;
 
@@ -509,140 +509,28 @@ install_packages_in_root (RpmOstreeTreeComposeContext  *self,
 }
 
 static gboolean
-parse_treefile_to_json (RpmOstreeTreeComposeContext  *self,
-                        const char    *treefile_path,
+parse_treefile_to_json (const char    *treefile_path,
+                        int            workdir_dfd,
+                        const char    *arch,
+                        RORTreefile  **out_treefile_rs,
                         JsonParser   **out_parser,
                         GError       **error)
 {
   g_autoptr(JsonParser) parser = json_parser_new ();
-  if (g_str_has_suffix (treefile_path, ".yaml") ||
-      g_str_has_suffix (treefile_path, ".yml"))
-    {
-      const char *arch = self ? dnf_context_get_base_arch (rpmostree_context_get_dnf (self->corectx)) : NULL;
-      g_autoptr(RORTreefile) tf = ror_treefile_new (treefile_path, arch,
-                                                    self->workdir_dfd,
-                                                    error);
-      if (!tf)
-        return glnx_prefix_error (error, "Failed to load YAML treefile");
+  g_autoptr(RORTreefile) treefile_rs = ror_treefile_new (treefile_path, arch, workdir_dfd, error);
+  if (!treefile_rs)
+    return glnx_prefix_error (error, "Failed to load YAML treefile");
 
-      glnx_fd_close int json_fd = ror_treefile_to_json (tf, error);
-      if (json_fd < 0)
-        return FALSE;
-      g_autoptr(GInputStream) json_s = g_unix_input_stream_new (json_fd, FALSE);
+  glnx_fd_close int json_fd = ror_treefile_to_json (treefile_rs, error);
+  if (json_fd < 0)
+    return FALSE;
+  g_autoptr(GInputStream) json_s = g_unix_input_stream_new (json_fd, FALSE);
 
-      if (!json_parser_load_from_stream (parser, json_s, NULL, error))
-        return FALSE;
-
-      /* We have first-one-wins semantics for this until we move all of the
-       * parsing into Rust.
-       */
-      if (!self->treefile_rs)
-        self->treefile_rs = g_steal_pointer (&tf);
-    }
-  else
-    {
-      if (!json_parser_load_from_file (parser, treefile_path, error))
-        return FALSE;
-    }
-
-  *out_parser = g_steal_pointer (&parser);
-  return TRUE;
-}
-
-
-static gboolean
-process_includes (RpmOstreeTreeComposeContext  *self,
-                  GFile             *treefile_path,
-                  guint              depth,
-                  JsonObject        *root,
-                  GCancellable      *cancellable,
-                  GError           **error)
-{
-  const guint maxdepth = 50;
-  if (depth > maxdepth)
-    return glnx_throw (error, "Exceeded maximum include depth of %u", maxdepth);
-
-  {
-    g_autoptr(GFile) parent = g_file_get_parent (treefile_path);
-    gboolean existed = FALSE;
-    if (self->treefile_context_dirs->len > 0)
-      {
-        GFile *prev = self->treefile_context_dirs->pdata[self->treefile_context_dirs->len-1];
-        if (g_file_equal (parent, prev))
-          existed = TRUE;
-      }
-    if (!existed)
-      {
-        g_ptr_array_add (self->treefile_context_dirs, parent);
-        parent = NULL; /* Transfer ownership */
-      }
-  }
-
-  const char *include_path;
-  if (!_rpmostree_jsonutil_object_get_optional_string_member (root, "include", &include_path, error))
+  if (!json_parser_load_from_stream (parser, json_s, NULL, error))
     return FALSE;
 
-  if (include_path)
-    {
-      g_autoptr(GFile) treefile_dirpath = g_file_get_parent (treefile_path);
-      g_autoptr(GFile) parent_path = g_file_resolve_relative_path (treefile_dirpath, include_path);
-
-      g_autoptr(JsonParser) parent_parser = NULL;
-      if (!parse_treefile_to_json (self,
-                                   gs_file_get_path_cached (parent_path),
-                                   &parent_parser, error))
-        return FALSE;
-
-      JsonNode *parent_rootval = json_parser_get_root (parent_parser);
-      if (!JSON_NODE_HOLDS_OBJECT (parent_rootval))
-        return glnx_throw (error, "Treefile root is not an object");
-      JsonObject *parent_root = json_node_get_object (parent_rootval);
-
-      if (!process_includes (self, parent_path, depth + 1, parent_root,
-                             cancellable, error))
-        return FALSE;
-
-      GList *members = json_object_get_members (parent_root);
-      for (GList *iter = members; iter; iter = iter->next)
-        {
-          const char *name = iter->data;
-          JsonNode *parent_val = json_object_get_member (parent_root, name);
-          JsonNode *val = json_object_get_member (root, name);
-
-          g_assert (parent_val);
-
-          if (!val)
-            json_object_set_member (root, name, json_node_copy (parent_val));
-          else
-            {
-              JsonNodeType parent_type =
-                json_node_get_node_type (parent_val);
-              JsonNodeType child_type =
-                json_node_get_node_type (val);
-              if (parent_type != child_type)
-                return glnx_throw (error, "Conflicting element type of '%s'", name);
-              if (child_type == JSON_NODE_ARRAY)
-                {
-                  JsonArray *parent_array = json_node_get_array (parent_val);
-                  JsonArray *child_array = json_node_get_array (val);
-                  JsonArray *new_child = json_array_new ();
-                  guint len;
-
-                  len = json_array_get_length (parent_array);
-                  for (guint i = 0; i < len; i++)
-                    json_array_add_element (new_child, json_node_copy (json_array_get_element (parent_array, i)));
-                  len = json_array_get_length (child_array);
-                  for (guint i = 0; i < len; i++)
-                    json_array_add_element (new_child, json_node_copy (json_array_get_element (child_array, i)));
-
-                  json_object_set_array_member (root, name, new_child);
-                }
-            }
-        }
-
-      json_object_remove_member (root, "include");
-    }
-
+  *out_parser = g_steal_pointer (&parser);
+  *out_treefile_rs = g_steal_pointer (&treefile_rs);
   return TRUE;
 }
 
@@ -739,8 +627,6 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
         return FALSE;
     }
 
-  self->treefile_context_dirs = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
-
   self->treefile_path = g_file_new_for_path (treefile_pathstr);
 
   if (opt_cachedir)
@@ -792,8 +678,11 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
   if (!self->corectx)
     return FALSE;
 
-  if (!parse_treefile_to_json (self, gs_file_get_path_cached (self->treefile_path),
-                               &self->treefile_parser, error))
+  const char *arch = dnf_context_get_base_arch (rpmostree_context_get_dnf (self->corectx));
+  if (!parse_treefile_to_json (gs_file_get_path_cached (self->treefile_path),
+                               self->workdir_dfd, arch,
+                               &self->treefile_rs, &self->treefile_parser,
+                               error))
     return FALSE;
 
   self->treefile_rootval = json_parser_get_root (self->treefile_parser);
@@ -801,11 +690,6 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
     return glnx_throw (error, "Treefile root is not an object");
   self->treefile = json_node_get_object (self->treefile_rootval);
 
-  if (!process_includes (self, self->treefile_path, 0, self->treefile,
-                         cancellable, error))
-    return FALSE;
-
-
   g_autoptr(GHashTable) varsubsts = rpmostree_dnfcontext_get_varsubsts (rpmostree_context_get_dnf (self->corectx));
   const char *input_ref = _rpmostree_jsonutil_object_require_string_member (self->treefile, "ref", error);
   if (!input_ref)
@@ -1014,8 +898,7 @@ impl_install_tree (RpmOstreeTreeComposeContext *self,
     return FALSE;
 
   /* Start postprocessing */
-  g_assert_cmpint (self->treefile_context_dirs->len, >, 0);
-  if (!rpmostree_treefile_postprocessing (self->rootfs_dfd, self->treefile_context_dirs->pdata[0],
+  if (!rpmostree_treefile_postprocessing (self->rootfs_dfd, self->treefile_rs,
                                           self->serialized_treefile, self->treefile,
                                           next_version, opt_unified_core,
                                           cancellable, error))
@@ -1385,9 +1268,14 @@ rpmostree_compose_builtin_postprocess (int             argc,
   const char *treefile_path = argc > 2 ? argv[2] : NULL;
   glnx_unref_object JsonParser *treefile_parser = NULL;
   JsonObject *treefile = NULL; /* Owned by parser */
+  g_autoptr(RORTreefile) treefile_rs = NULL;
+  g_auto(GLnxTmpDir) workdir_tmp = { 0, };
   if (treefile_path)
     {
-      if (!parse_treefile_to_json (NULL, treefile_path, &treefile_parser, error))
+      if (!glnx_mkdtempat (AT_FDCWD, "/var/tmp/rpm-ostree.XXXXXX", 0700, &workdir_tmp, error))
+        return FALSE;
+      if (!parse_treefile_to_json (treefile_path, workdir_tmp.fd, NULL,
+                                   &treefile_rs, &treefile_parser, error))
         return FALSE;
 
       JsonNode *treefile_rootval = json_parser_get_root (treefile_parser);

src/app/rpmostree-composeutil.c

@@ -49,7 +49,7 @@
 gboolean
 rpmostree_composeutil_checksum (GBytes            *serialized_treefile,
                                 HyGoal             goal,
-                                GFile             *contextdir,
+                                RORTreefile       *tf,
                                 JsonArray         *add_files,
                                 char             **out_checksum,
                                 GError           **error)
@@ -71,31 +71,19 @@ rpmostree_composeutil_checksum (GBytes            *serialized_treefile,
       guint i, len = json_array_get_length (add_files);
       for (i = 0; i < len; i++)
         {
-          g_autoptr(GFile) srcfile = NULL;
-          const char *src, *dest;
           JsonArray *add_el = json_array_get_array_element (add_files, i);
-
           if (!add_el)
             return glnx_throw (error, "Element in add-files is not an array");
-
-          src = _rpmostree_jsonutil_array_require_string_element (add_el, 0, error);
+          const char *src = _rpmostree_jsonutil_array_require_string_element (add_el, 0, error);
           if (!src)
             return FALSE;
 
-          dest = _rpmostree_jsonutil_array_require_string_element (add_el, 1, error);
-          if (!dest)
-            return FALSE;
-
-          srcfile = g_file_resolve_relative_path (contextdir, src);
-
-          if (!_rpmostree_util_update_checksum_from_file (checksum,
-                                                          AT_FDCWD,
-                                                          gs_file_get_path_cached (srcfile),
-                                                          NULL,
-                                                          error))
-            return FALSE;
-
-          g_checksum_update (checksum, (const guint8 *) dest, strlen (dest));
+          int src_fd = ror_treefile_get_add_file_fd (tf, src);
+          g_assert_cmpint (src_fd, !=, -1);
+          g_autoptr(GBytes) bytes = glnx_fd_readall_bytes (src_fd, NULL, FALSE);
+          gsize len;
+          const guint8* buf = g_bytes_get_data (bytes, &len);
+          g_checksum_update (checksum, (const guint8 *) buf, len);
         }
 
     }
@@ -151,34 +139,24 @@ rpmostree_composeutil_legacy_prep_dev (int         rootfs_dfd,
   return TRUE;
 }
 
+
 gboolean
-rpmostree_composeutil_sanity_checks (JsonObject   *treedata,
-                                     GFile        *contextdir,
+rpmostree_composeutil_sanity_checks (RORTreefile  *tf,
+                                     JsonObject   *treefile,
                                      GCancellable *cancellable,
                                      GError      **error)
 {
-  /* Check that postprocess-script is executable; https://github.com/projectatomic/rpm-ostree/issues/817 */
-  const char *postprocess_script = NULL;
+  int fd = ror_treefile_get_postprocess_script_fd (tf);
+  if (fd != -1)
+    {
+      /* Check that postprocess-script is executable; https://github.com/projectatomic/rpm-ostree/issues/817 */
+      struct stat stbuf;
+      if (!glnx_fstat (fd, &stbuf, error))
+        return glnx_prefix_error (error, "postprocess-script");
 
-  if (!_rpmostree_jsonutil_object_get_optional_string_member (treedata, "postprocess-script",
-                                                              &postprocess_script, error))
-    return FALSE;
-
-  if (!postprocess_script)
-    return TRUE;
-
-  g_autofree char *src = NULL;
-  if (g_path_is_absolute (postprocess_script))
-    src = g_strdup (postprocess_script);
-  else
-    src = g_build_filename (gs_file_get_path_cached (contextdir), postprocess_script, NULL);
-
-  struct stat stbuf;
-  if (!glnx_fstatat (AT_FDCWD, src, &stbuf, 0, error))
-    return glnx_prefix_error (error, "postprocess-script");
-
-  if ((stbuf.st_mode & S_IXUSR) == 0)
-    return glnx_throw (error, "postprocess-script (%s) must be executable", postprocess_script);
+      if ((stbuf.st_mode & S_IXUSR) == 0)
+        return glnx_throw (error, "postprocess-script must be executable");
+    }
 
   /* Insert other sanity checks here */
 

src/app/rpmostree-composeutil.h

@@ -23,13 +23,14 @@
 #include <ostree.h>
 
 #include "rpmostree-core.h"
+#include "rpmostree-rust.h"
 
 G_BEGIN_DECLS
 
 gboolean
 rpmostree_composeutil_checksum (GBytes            *serialized_treefile,
                                 HyGoal             goal,
-                                GFile             *contextdir,
+                                RORTreefile       *tf,
                                 JsonArray         *add_files,
                                 char             **out_checksum,
                                 GError           **error);
@@ -39,8 +40,8 @@ rpmostree_composeutil_legacy_prep_dev (int         rootfs_dfd,
                                        GError    **error);
 
 gboolean
-rpmostree_composeutil_sanity_checks (JsonObject   *treedata,
-                                     GFile        *contextdir,
+rpmostree_composeutil_sanity_checks (RORTreefile  *tf,
+                                     JsonObject   *treefile,
                                      GCancellable *cancellable,
                                      GError      **error);
 

src/libpriv/rpmostree-postprocess.c

@@ -1341,7 +1341,7 @@ rpmostree_rootfs_postprocess_common (int           rootfs_fd,
  */
 static gboolean
 copy_additional_files (int            rootfs_dfd,
-                       GFile         *context_directory,
+                       RORTreefile   *treefile_rs,
                        JsonObject    *treefile,
                        GCancellable  *cancellable,
                        GError       **error)
@@ -1356,11 +1356,6 @@ copy_additional_files (int            rootfs_dfd,
   else
     return TRUE; /* Early return */
 
-  glnx_autofd int context_dfd = -1;
-  if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (context_directory), TRUE,
-                       &context_dfd, error))
-    return FALSE;
-
   /* Reusable dirname buffer */
   g_autoptr(GString) dnbuf = g_string_new ("");
   for (guint i = 0; i < len; i++)
@@ -1403,12 +1398,26 @@ copy_additional_files (int            rootfs_dfd,
       if (!glnx_shutil_mkdir_p_at (rootfs_dfd, dn, 0755, cancellable, error))
         return FALSE;
 
-      /* FIXME: Should probably use GLNX_FILE_COPY_NOXATTRS, but someone
-       * may be relying on current semantics?
+      int src_fd = ror_treefile_get_add_file_fd (treefile_rs, src);
+      g_assert_cmpint (src_fd, !=, -1);
+
+      g_auto(GLnxTmpfile) tmpf = { 0, };
+      if (!glnx_open_tmpfile_linkable_at (rootfs_dfd, ".", O_CLOEXEC | O_WRONLY, &tmpf, error))
+        return FALSE;
+      if (glnx_regfile_copy_bytes (src_fd, tmpf.fd, (off_t)-1) < 0)
+        return glnx_throw_errno_prefix (error, "regfile copy");
+      struct stat src_stbuf;
+      if (!glnx_fstat (src_fd, &src_stbuf, error))
+        return FALSE;
+      if (!glnx_fchmod (tmpf.fd, src_stbuf.st_mode, error))
+        return FALSE;
+      /* Note we used to copy xattrs here, we no longer do.  Hopefully
+       * no one breaks.
        */
-      if (!glnx_file_copy_at (context_dfd, src, NULL, rootfs_dfd, dest, 0,
-                              cancellable, error))
-          return glnx_prefix_error (error, "Copying file '%s' into target", src);
+      if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE,
+                                 rootfs_dfd, dest,
+                                 error))
+        return FALSE;
     }
 
   return TRUE;
@@ -1464,7 +1473,7 @@ mutate_os_release (const char    *contents,
  */
 gboolean
 rpmostree_treefile_postprocessing (int            rootfs_fd,
-                                   GFile         *context_directory,
+                                   RORTreefile   *treefile_rs,
                                    GBytes        *serialized_treefile,
                                    JsonObject    *treefile,
                                    const char    *next_version,
@@ -1472,7 +1481,7 @@ rpmostree_treefile_postprocessing (int            rootfs_fd,
                                    GCancellable  *cancellable,
                                    GError       **error)
 {
-  g_assert (context_directory);
+  g_assert (treefile_rs);
   g_assert (treefile);
 
   if (!rename_if_exists (rootfs_fd, "etc", rootfs_fd, "usr/etc", error))
@@ -1720,7 +1729,7 @@ rpmostree_treefile_postprocessing (int            rootfs_fd,
   }
 
   /* Copy in additional files before postprocessing */
-  if (!copy_additional_files (rootfs_fd, context_directory, treefile, cancellable, error))
+  if (!copy_additional_files (rootfs_fd, treefile_rs, treefile, cancellable, error))
     return FALSE;
 
   if (json_object_has_member (treefile, "postprocess"))
@@ -1753,43 +1762,38 @@ rpmostree_treefile_postprocessing (int            rootfs_fd,
         }
     }
 
-  const char *postprocess_script = NULL;
-  if (!_rpmostree_jsonutil_object_get_optional_string_member (treefile, "postprocess-script",
-                                                              &postprocess_script, error))
-    return FALSE;
-  if (postprocess_script)
+  int postprocess_script_fd = ror_treefile_get_postprocess_script_fd (treefile_rs);
+  if (postprocess_script_fd != -1)
     {
-      const char *bn = glnx_basename (postprocess_script);
-      g_autofree char *src = NULL;
-      g_autofree char *binpath = NULL;
-
-      if (g_path_is_absolute (postprocess_script))
-        src = g_strdup (postprocess_script);
-      else
-        src = g_build_filename (gs_file_get_path_cached (context_directory), postprocess_script, NULL);
-
-      binpath = g_strconcat ("/usr/bin/rpmostree-postprocess-", bn, NULL);
-      /* Clone all the things */
-
-      /* Note we need to make binpath *not* absolute here */
+      const char *binpath = "/usr/bin/rpmostree-treefile-postprocess-script";
       const char *target_binpath = binpath + 1;
-      g_assert_cmpint (*target_binpath, !=, '/');
-      if (!glnx_file_copy_at (AT_FDCWD, src, NULL, rootfs_fd, target_binpath,
-                              GLNX_FILE_COPY_NOXATTRS, cancellable, error))
+      g_auto(GLnxTmpfile) tmpf = { 0, };
+      if (!glnx_open_tmpfile_linkable_at (rootfs_fd, ".", O_CLOEXEC | O_WRONLY, &tmpf, error))
+        return FALSE;
+      if (glnx_regfile_copy_bytes (postprocess_script_fd, tmpf.fd, (off_t)-1) < 0)
+        return glnx_throw_errno_prefix (error, "regfile copy");
+      if (!glnx_fchmod (tmpf.fd, 0755, error))
+        return FALSE;
+      if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE,
+                                 rootfs_fd, target_binpath,
+                                 error))
         return FALSE;
 
-      g_print ("Executing postprocessing script '%s'\n", bn);
+      /* Can't have a writable fd open when we go to exec */
+      glnx_tmpfile_clear (&tmpf);
+
+      g_print ("Executing postprocessing script\n");
 
       {
-        char *child_argv[] = { binpath, NULL };
+        char *child_argv[] = { (char*)binpath, NULL };
         if (!run_bwrap_mutably (rootfs_fd, binpath, child_argv, unified_core_mode, cancellable, error))
-          return glnx_prefix_error (error, "While executing postprocessing script '%s'", bn);
+          return glnx_prefix_error (error, "While executing postprocessing script");
       }
 
       if (!glnx_unlinkat (rootfs_fd, target_binpath, 0, error))
         return FALSE;
 
-      g_print ("Finished postprocessing script '%s'\n", bn);
+      g_print ("Finished postprocessing script\n");
     }
 
   return TRUE;

src/libpriv/rpmostree-postprocess.h

@@ -22,6 +22,7 @@
 
 #include <ostree.h>
 #include "rpmostree-json-parsing.h"
+#include "rpmostree-rust.h"
 
 /* "public" for unit tests */
 char *
@@ -30,7 +31,7 @@ rpmostree_postprocess_replace_nsswitch (const char *buf,
 
 gboolean
 rpmostree_treefile_postprocessing (int            rootfs_fd,
-                                   GFile         *context_directory,
+                                   RORTreefile   *treefile_rs,
                                    GBytes        *serialized_treefile,
                                    JsonObject    *treefile,
                                    const char    *next_version,