From e3be475566f18705c89cbc0d68141fd7e9f8f4dc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 22 Sep 2018 08:10:21 -0400 Subject: [PATCH] compose+rust: Parse includes via Rust too This follows up to https://github.com/projectatomic/rpm-ostree/pull/1576 AKA commit 2e567840ca4dd1c5f28392b3f9c077bcdbaebd3f - we now process treefile inheritance in Rust code. Previously for elements which reference external files (`postprocess-script` and `add-files`) we'd hardcoded things to only look in the first context dir. Now we open file descriptors in the Rust side for these "externals" as we're parsing, and load them C side. Hence we'll correctly handle a `postprocess-script` from an included config. Other advantages are that the include handling was ugly un-typesafe C code with no unit tests, now it's memory safe Rust with unit tests. The downside here is I ended up spelling out the list of fields again - there's probably a way to unify this via macros but for now I think this is OK. Closes: #1574 Approved by: jlebon --- Makefile-lib.am | 2 +- rust/src/lib.rs | 38 +++- rust/src/treefile.rs | 253 ++++++++++++++++++++--- src/app/rpmostree-compose-builtin-tree.c | 178 +++------------- src/app/rpmostree-composeutil.c | 64 ++---- src/app/rpmostree-composeutil.h | 7 +- src/libpriv/rpmostree-postprocess.c | 82 ++++---- src/libpriv/rpmostree-postprocess.h | 3 +- 8 files changed, 368 insertions(+), 259 deletions(-) diff --git a/Makefile-lib.am b/Makefile-lib.am index ca1028cd..53f4d9b7 100644 --- a/Makefile-lib.am +++ b/Makefile-lib.am @@ -34,7 +34,7 @@ librpmostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/libglnx -I$(srcdir)/src/libp -fvisibility=hidden '-D_RPMOSTREE_EXTERN=__attribute((visibility("default"))) extern' \ $(PKGDEP_RPMOSTREE_CFLAGS) librpmostree_1_la_LDFLAGS = $(AM_LDFLAGS) -version-number 1:0:0 -Bsymbolic-functions -librpmostree_1_la_LIBADD = $(PKGDEP_RPMOSTREE_LIBS) librpmostreepriv.la +librpmostree_1_la_LIBADD = $(PKGDEP_RPMOSTREE_LIBS) librpmostreepriv.la $(librpmostree_rust_path) # bundled libdnf INTROSPECTION_SCANNER_ENV = env LD_LIBRARY_PATH=$(top_builddir)/libdnf-build/libdnf diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 6f2fa961..d181cf5d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -32,8 +32,9 @@ extern crate serde_json; extern crate serde_yaml; use std::ffi::{CStr, OsStr}; +use std::io::Seek; use std::os::unix::ffi::OsStrExt; -use std::os::unix::io::{FromRawFd, IntoRawFd}; +use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; use std::{io, ptr}; mod glibutils; @@ -94,6 +95,41 @@ pub extern "C" fn ror_treefile_new( ) } +#[no_mangle] +pub extern "C" fn ror_treefile_get_dfd(tf: *mut Treefile) -> libc::c_int { + assert!(!tf.is_null()); + let tf = unsafe { &mut *tf }; + tf.primary_dfd.as_raw_fd() +} + +#[no_mangle] +pub extern "C" fn ror_treefile_get_postprocess_script_fd(tf: *mut Treefile) -> libc::c_int { + assert!(!tf.is_null()); + let tf = unsafe { &mut *tf }; + if let Some(ref mut fd) = tf.externals.postprocess_script.as_ref() { + // We always seek to the start + fd.seek(io::SeekFrom::Start(0)).unwrap(); + fd.as_raw_fd() + } else { + -1 + } +} + +#[no_mangle] +pub extern "C" fn ror_treefile_get_add_file_fd( + tf: *mut Treefile, + filename: *const libc::c_char, +) -> libc::c_int { + assert!(!tf.is_null()); + let tf = unsafe { &mut *tf }; + let filename = OsStr::from_bytes(bytes_from_nonnull(filename)); + let filename = filename.to_string_lossy().into_owned(); + let mut fd = tf.externals.add_files.get(&filename).expect("add-file"); + // We always seek to the start + fd.seek(io::SeekFrom::Start(0)).unwrap(); + fd.as_raw_fd() +} + #[no_mangle] pub extern "C" fn ror_treefile_to_json( tf: *mut Treefile, diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index 6e62b947..a0088097 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -26,15 +26,31 @@ use serde_json; use serde_yaml; use std::io::prelude::*; use std::path::Path; -use std::{fs, io}; +use std::{collections, fs, io}; use tempfile; const ARCH_X86_64: &'static str = "x86_64"; +const INCLUDE_MAXDEPTH: u32 = 50; + +/// This struct holds file descriptors for any external files/data referenced by +/// a TreeComposeConfig. +pub struct TreefileExternals { + pub postprocess_script: Option, + pub add_files: collections::HashMap, +} pub struct Treefile { pub workdir: openat::Dir, + pub primary_dfd: openat::Dir, pub parsed: TreeComposeConfig, pub rojig_spec: Option, + pub externals: TreefileExternals, +} + +// We only use this while parsing +struct ConfigAndExternals { + config: TreeComposeConfig, + externals: TreefileExternals, } enum InputFormat { @@ -43,6 +59,7 @@ enum InputFormat { } /// Parse a YAML treefile definition using architecture `arch`. +/// This does not open the externals. fn treefile_parse_stream( fmt: InputFormat, input: &mut R, @@ -100,32 +117,175 @@ fn treefile_parse_stream( Ok(treefile) } +/// Given a treefile filename and an architecture, parse it and also +/// open its external files. +fn treefile_parse>( + filename: P, + arch: Option<&str>, +) -> io::Result { + let filename = filename.as_ref(); + let mut f = io::BufReader::new(fs::File::open(filename)?); + let basename = filename + .file_name() + .map(|s| s.to_string_lossy()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Expected a filename"))?; + let fmt = if basename.ends_with(".yaml") || basename.ends_with(".yml") { + InputFormat::YAML + } else { + InputFormat::JSON + }; + let tf = treefile_parse_stream(fmt, &mut f, arch).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Parsing {}: {}", filename.to_string_lossy(), e.to_string()), + ) + })?; + let postprocess_script = if let Some(ref postprocess) = tf.postprocess_script.as_ref() { + Some(fs::File::open(filename.with_file_name(postprocess))?) + } else { + None + }; + let mut add_files: collections::HashMap = collections::HashMap::new(); + if let Some(ref add_file_names) = tf.add_files.as_ref() { + for (name, _) in add_file_names.iter() { + add_files.insert(name.clone(), fs::File::open(filename.with_file_name(name))?); + } + } + Ok(ConfigAndExternals { + config: tf, + externals: TreefileExternals { + postprocess_script, + add_files, + }, + }) +} + +/// Merge a "basic" or non-array field. The semantics originally defined for +/// these are that first-one wins. +fn merge_basic_field(dest: &mut Option, src: &mut Option) { + if dest.is_some() { + return; + } + *dest = src.take() +} + +/// Merge a vector field by appending. This semantic was originally designed for +/// the `packages` key. +fn merge_vec_field(dest: &mut Option>, src: &mut Option>) { + if let Some(mut srcv) = src.take() { + if let Some(ref mut destv) = dest { + srcv.append(destv); + } + *dest = Some(srcv); + } +} + +/// Given two configs, merge them. Ideally we'd do some macro magic and avoid +/// listing all of the fields again. +fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) { + merge_basic_field(&mut dest.treeref, &mut src.treeref); + merge_basic_field(&mut dest.rojig, &mut src.rojig); + merge_vec_field(&mut dest.repos, &mut src.repos); + merge_basic_field(&mut dest.selinux, &mut src.selinux); + merge_basic_field(&mut dest.gpg_key, &mut src.gpg_key); + merge_basic_field(&mut dest.include, &mut src.include); + merge_vec_field(&mut dest.packages, &mut src.packages); + merge_basic_field(&mut dest.recommends, &mut src.recommends); + merge_basic_field(&mut dest.documentation, &mut src.documentation); + merge_vec_field(&mut dest.install_langs, &mut src.install_langs); + merge_vec_field(&mut dest.initramfs_args, &mut src.initramfs_args); + merge_basic_field(&mut dest.boot_location, &mut src.boot_location); + merge_basic_field(&mut dest.tmp_is_dir, &mut src.tmp_is_dir); + merge_basic_field(&mut dest.default_target, &mut src.default_target); + merge_vec_field(&mut dest.units, &mut src.units); + merge_basic_field(&mut dest.machineid_compat, &mut src.machineid_compat); + merge_basic_field(&mut dest.releasever, &mut src.releasever); + merge_basic_field( + &mut dest.automatic_version_prefix, + &mut src.automatic_version_prefix, + ); + merge_basic_field(&mut dest.mutate_os_release, &mut src.mutate_os_release); + merge_basic_field(&mut dest.etc_group_members, &mut src.etc_group_members); + merge_basic_field(&mut dest.preserve_passwd, &mut src.preserve_passwd); + merge_basic_field(&mut dest.check_passwd, &mut src.check_passwd); + merge_basic_field(&mut dest.check_groups, &mut src.check_groups); + merge_basic_field( + &mut dest.ignore_removed_users, + &mut src.ignore_removed_users, + ); + merge_basic_field( + &mut dest.ignore_removed_groups, + &mut src.ignore_removed_groups, + ); + merge_basic_field(&mut dest.postprocess_script, &mut src.postprocess_script); + merge_vec_field(&mut dest.postprocess, &mut src.postprocess); + merge_vec_field(&mut dest.add_files, &mut src.add_files); + merge_vec_field(&mut dest.remove_files, &mut src.remove_files); + merge_vec_field( + &mut dest.remove_from_packages, + &mut src.remove_from_packages, + ); +} + +/// Merge the treefile externals. There are currently only two keys that +/// reference external files. +fn treefile_merge_externals(dest: &mut TreefileExternals, src: &mut TreefileExternals) { + // This one, being a basic-valued field, has first-wins semantics. + if dest.postprocess_script.is_none() { + dest.postprocess_script = src.postprocess_script.take(); + } + + // add-files is an array and hence has append semantics. + for (k, v) in src.add_files.drain() { + dest.add_files.insert(k, v); + } +} + +/// Recursively parse a treefile, merging along the way. +fn treefile_parse_recurse>( + filename: P, + arch: Option<&str>, + depth: u32, +) -> io::Result { + let filename = filename.as_ref(); + let mut parsed = treefile_parse(filename, arch)?; + let include_path = parsed.config.include.take(); + if let &Some(ref include_path) = &include_path { + if depth == INCLUDE_MAXDEPTH { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("Reached maximum include depth {}", INCLUDE_MAXDEPTH), + )); + } + let parent = filename.parent().unwrap(); + let include_path = parent.join(include_path); + let mut included = treefile_parse_recurse(include_path, arch, depth + 1)?; + treefile_merge(&mut parsed.config, &mut included.config); + treefile_merge_externals(&mut parsed.externals, &mut included.externals); + } + Ok(parsed) +} + impl Treefile { + /// The main treefile creation entrypoint. pub fn new_boxed( filename: &Path, arch: Option<&str>, workdir: openat::Dir, ) -> io::Result> { - let mut f = io::BufReader::new(fs::File::open(filename)?); - let basename = filename - .file_name() - .map(|s| s.to_string_lossy()) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Expected a filename"))?; - let fmt = if basename.ends_with(".yaml") || basename.ends_with(".yml") { - InputFormat::YAML - } else { - InputFormat::JSON - }; - let parsed = treefile_parse_stream(fmt, &mut f, arch)?; - let rojig_spec = if let &Some(ref rojig) = &parsed.rojig { + let parsed = treefile_parse_recurse(filename, arch, 0)?; + let dfd = openat::Dir::open(filename.parent().unwrap())?; + let rojig_spec = if let &Some(ref rojig) = &parsed.config.rojig { Some(Treefile::write_rojig_spec(&workdir, rojig)?) } else { None }; Ok(Box::new(Treefile { - parsed: parsed, + primary_dfd: dfd, + parsed: parsed.config, workdir: workdir, rojig_spec: rojig_spec, + externals: parsed.externals, })) } @@ -139,6 +299,7 @@ impl Treefile { Ok(tmpf) } + /// Generate a rojig spec file. fn write_rojig_spec<'a, 'b>(workdir: &'a openat::Dir, r: &'b Rojig) -> io::Result { let description = r .description @@ -186,6 +347,8 @@ for x in *; do mv ${{x}} %{{buildroot}}%{{_prefix}}/lib/ostree-jigdo/%{{name}}; } } +/// For increased readability in YAML/JSON, we support whitespace in individual +/// array elements. fn whitespace_split_packages(pkgs: &[String]) -> Vec { pkgs.iter() .flat_map(|pkg| pkg.split_whitespace().map(String::from)) @@ -390,11 +553,6 @@ packages-x86_64: - grub2 grub2-tools packages-s390x: - zipl -add-files: - - - foo - - /usr/bin/foo - - - baz - - /usr/bin/blah "###; // This one has "comments" (hence unknown keys) @@ -416,7 +574,29 @@ add-files: treefile_parse_stream(InputFormat::YAML, &mut input, Some(ARCH_X86_64)).unwrap(); assert!(treefile.treeref.unwrap() == "exampleos/x86_64/blah"); assert!(treefile.packages.unwrap().len() == 5); + } + + #[test] + fn basic_valid_add_remove_files() { + let mut buf = VALID_PRELUDE.to_string(); + buf.push_str( + r###" +add-files: + - - foo + - /usr/bin/foo + - - baz + - /usr/bin/blah +remove-files: + - foo + - bar +"###, + ); + let buf = buf.as_bytes(); + let mut input = io::BufReader::new(buf); + let treefile = + treefile_parse_stream(InputFormat::YAML, &mut input, Some(ARCH_X86_64)).unwrap(); assert!(treefile.add_files.unwrap().len() == 2); + assert!(treefile.remove_files.unwrap().len() == 2); } #[test] @@ -496,17 +676,17 @@ add-files: assert!(tf.parsed.machineid_compat.is_none()); } - #[test] - fn test_treefile_new_rojig() { - let mut buf = VALID_PRELUDE.to_string(); - buf.push_str( - r###" + const ROJIG_YAML: &'static str = r###" rojig: name: "exampleos" license: "MIT" summary: "ExampleOS rojig base image" -"###, - ); +"###; + + #[test] + fn test_treefile_new_rojig() { + let mut buf = VALID_PRELUDE.to_string(); + buf.push_str(ROJIG_YAML); let t = TreefileTest::new(buf.as_str(), None).unwrap(); let tf = &t.tf; let rojig = tf.parsed.rojig.as_ref().unwrap(); @@ -516,4 +696,25 @@ rojig: assert!(rojig_spec.file_name().unwrap() == "exampleos.spec"); } + #[test] + fn test_treefile_merge() { + let arch = Some(ARCH_X86_64); + let mut base_input = io::BufReader::new(VALID_PRELUDE.as_bytes()); + let mut base = treefile_parse_stream(InputFormat::YAML, &mut base_input, arch).unwrap(); + let mut mid_input = io::BufReader::new( + r###" +packages: + - some layered packages +"###.as_bytes(), + ); + let mut mid = treefile_parse_stream(InputFormat::YAML, &mut mid_input, arch).unwrap(); + let mut top_input = io::BufReader::new(ROJIG_YAML.as_bytes()); + let mut top = treefile_parse_stream(InputFormat::YAML, &mut top_input, arch).unwrap(); + treefile_merge(&mut mid, &mut base); + treefile_merge(&mut top, &mut mid); + let tf = ⊤ + assert!(tf.packages.as_ref().unwrap().len() == 8); + let rojig = tf.rojig.as_ref().unwrap(); + assert!(rojig.name == "exampleos"); + } } diff --git a/src/app/rpmostree-compose-builtin-tree.c b/src/app/rpmostree-compose-builtin-tree.c index c2e2c0f9..a5b5b99e 100644 --- a/src/app/rpmostree-compose-builtin-tree.c +++ b/src/app/rpmostree-compose-builtin-tree.c @@ -105,8 +105,6 @@ static GOptionEntry commit_option_entries[] = { }; typedef struct { - GPtrArray *treefile_context_dirs; - RpmOstreeContext *corectx; GFile *treefile_path; GHashTable *metadata; @@ -133,7 +131,6 @@ typedef struct { static void rpm_ostree_tree_compose_context_free (RpmOstreeTreeComposeContext *ctx) { - g_clear_pointer (&ctx->treefile_context_dirs, (GDestroyNotify)g_ptr_array_unref); g_clear_object (&ctx->corectx); g_clear_object (&ctx->treefile_path); g_clear_pointer (&ctx->metadata, g_hash_table_unref); @@ -263,8 +260,10 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, rpmlogSetFile(NULL); } - GFile *contextdir = self->treefile_context_dirs->pdata[0]; - dnf_context_set_repo_dir (dnfctx, gs_file_get_path_cached (contextdir)); + { int tf_dfd = ror_treefile_get_dfd (self->treefile_rs); + g_autofree char *abs_tf_path = glnx_fdrel_abspath (tf_dfd, "."); + dnf_context_set_repo_dir (dnfctx, abs_tf_path); + } /* By default, retain packages in addition to metadata with --cachedir, unless * we're doing unified core, in which case the pkgcache repo is the cache. But @@ -374,7 +373,7 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, g_autofree char *ret_new_inputhash = NULL; if (!rpmostree_composeutil_checksum (self->serialized_treefile, dnf_context_get_goal (dnfctx), - contextdir, add_files, + self->treefile_rs, add_files, &ret_new_inputhash, error)) return FALSE; @@ -403,7 +402,8 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, if (opt_dry_run) return TRUE; /* NB: early return */ - if (!rpmostree_composeutil_sanity_checks (treedata, self->treefile_context_dirs->pdata[0], + if (!rpmostree_composeutil_sanity_checks (self->treefile_rs, + self->treefile, cancellable, error)) return FALSE; @@ -509,140 +509,28 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, } static gboolean -parse_treefile_to_json (RpmOstreeTreeComposeContext *self, - const char *treefile_path, +parse_treefile_to_json (const char *treefile_path, + int workdir_dfd, + const char *arch, + RORTreefile **out_treefile_rs, JsonParser **out_parser, GError **error) { g_autoptr(JsonParser) parser = json_parser_new (); - if (g_str_has_suffix (treefile_path, ".yaml") || - g_str_has_suffix (treefile_path, ".yml")) - { - const char *arch = self ? dnf_context_get_base_arch (rpmostree_context_get_dnf (self->corectx)) : NULL; - g_autoptr(RORTreefile) tf = ror_treefile_new (treefile_path, arch, - self->workdir_dfd, - error); - if (!tf) - return glnx_prefix_error (error, "Failed to load YAML treefile"); + g_autoptr(RORTreefile) treefile_rs = ror_treefile_new (treefile_path, arch, workdir_dfd, error); + if (!treefile_rs) + return glnx_prefix_error (error, "Failed to load YAML treefile"); - glnx_fd_close int json_fd = ror_treefile_to_json (tf, error); - if (json_fd < 0) - return FALSE; - g_autoptr(GInputStream) json_s = g_unix_input_stream_new (json_fd, FALSE); + glnx_fd_close int json_fd = ror_treefile_to_json (treefile_rs, error); + if (json_fd < 0) + return FALSE; + g_autoptr(GInputStream) json_s = g_unix_input_stream_new (json_fd, FALSE); - if (!json_parser_load_from_stream (parser, json_s, NULL, error)) - return FALSE; - - /* We have first-one-wins semantics for this until we move all of the - * parsing into Rust. - */ - if (!self->treefile_rs) - self->treefile_rs = g_steal_pointer (&tf); - } - else - { - if (!json_parser_load_from_file (parser, treefile_path, error)) - return FALSE; - } - - *out_parser = g_steal_pointer (&parser); - return TRUE; -} - - -static gboolean -process_includes (RpmOstreeTreeComposeContext *self, - GFile *treefile_path, - guint depth, - JsonObject *root, - GCancellable *cancellable, - GError **error) -{ - const guint maxdepth = 50; - if (depth > maxdepth) - return glnx_throw (error, "Exceeded maximum include depth of %u", maxdepth); - - { - g_autoptr(GFile) parent = g_file_get_parent (treefile_path); - gboolean existed = FALSE; - if (self->treefile_context_dirs->len > 0) - { - GFile *prev = self->treefile_context_dirs->pdata[self->treefile_context_dirs->len-1]; - if (g_file_equal (parent, prev)) - existed = TRUE; - } - if (!existed) - { - g_ptr_array_add (self->treefile_context_dirs, parent); - parent = NULL; /* Transfer ownership */ - } - } - - const char *include_path; - if (!_rpmostree_jsonutil_object_get_optional_string_member (root, "include", &include_path, error)) + if (!json_parser_load_from_stream (parser, json_s, NULL, error)) return FALSE; - if (include_path) - { - g_autoptr(GFile) treefile_dirpath = g_file_get_parent (treefile_path); - g_autoptr(GFile) parent_path = g_file_resolve_relative_path (treefile_dirpath, include_path); - - g_autoptr(JsonParser) parent_parser = NULL; - if (!parse_treefile_to_json (self, - gs_file_get_path_cached (parent_path), - &parent_parser, error)) - return FALSE; - - JsonNode *parent_rootval = json_parser_get_root (parent_parser); - if (!JSON_NODE_HOLDS_OBJECT (parent_rootval)) - return glnx_throw (error, "Treefile root is not an object"); - JsonObject *parent_root = json_node_get_object (parent_rootval); - - if (!process_includes (self, parent_path, depth + 1, parent_root, - cancellable, error)) - return FALSE; - - GList *members = json_object_get_members (parent_root); - for (GList *iter = members; iter; iter = iter->next) - { - const char *name = iter->data; - JsonNode *parent_val = json_object_get_member (parent_root, name); - JsonNode *val = json_object_get_member (root, name); - - g_assert (parent_val); - - if (!val) - json_object_set_member (root, name, json_node_copy (parent_val)); - else - { - JsonNodeType parent_type = - json_node_get_node_type (parent_val); - JsonNodeType child_type = - json_node_get_node_type (val); - if (parent_type != child_type) - return glnx_throw (error, "Conflicting element type of '%s'", name); - if (child_type == JSON_NODE_ARRAY) - { - JsonArray *parent_array = json_node_get_array (parent_val); - JsonArray *child_array = json_node_get_array (val); - JsonArray *new_child = json_array_new (); - guint len; - - len = json_array_get_length (parent_array); - for (guint i = 0; i < len; i++) - json_array_add_element (new_child, json_node_copy (json_array_get_element (parent_array, i))); - len = json_array_get_length (child_array); - for (guint i = 0; i < len; i++) - json_array_add_element (new_child, json_node_copy (json_array_get_element (child_array, i))); - - json_object_set_array_member (root, name, new_child); - } - } - } - - json_object_remove_member (root, "include"); - } - + *out_parser = g_steal_pointer (&parser); + *out_treefile_rs = g_steal_pointer (&treefile_rs); return TRUE; } @@ -739,8 +627,6 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, return FALSE; } - self->treefile_context_dirs = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - self->treefile_path = g_file_new_for_path (treefile_pathstr); if (opt_cachedir) @@ -792,8 +678,11 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, if (!self->corectx) return FALSE; - if (!parse_treefile_to_json (self, gs_file_get_path_cached (self->treefile_path), - &self->treefile_parser, error)) + const char *arch = dnf_context_get_base_arch (rpmostree_context_get_dnf (self->corectx)); + if (!parse_treefile_to_json (gs_file_get_path_cached (self->treefile_path), + self->workdir_dfd, arch, + &self->treefile_rs, &self->treefile_parser, + error)) return FALSE; self->treefile_rootval = json_parser_get_root (self->treefile_parser); @@ -801,11 +690,6 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, return glnx_throw (error, "Treefile root is not an object"); self->treefile = json_node_get_object (self->treefile_rootval); - if (!process_includes (self, self->treefile_path, 0, self->treefile, - cancellable, error)) - return FALSE; - - g_autoptr(GHashTable) varsubsts = rpmostree_dnfcontext_get_varsubsts (rpmostree_context_get_dnf (self->corectx)); const char *input_ref = _rpmostree_jsonutil_object_require_string_member (self->treefile, "ref", error); if (!input_ref) @@ -1014,8 +898,7 @@ impl_install_tree (RpmOstreeTreeComposeContext *self, return FALSE; /* Start postprocessing */ - g_assert_cmpint (self->treefile_context_dirs->len, >, 0); - if (!rpmostree_treefile_postprocessing (self->rootfs_dfd, self->treefile_context_dirs->pdata[0], + if (!rpmostree_treefile_postprocessing (self->rootfs_dfd, self->treefile_rs, self->serialized_treefile, self->treefile, next_version, opt_unified_core, cancellable, error)) @@ -1385,9 +1268,14 @@ rpmostree_compose_builtin_postprocess (int argc, const char *treefile_path = argc > 2 ? argv[2] : NULL; glnx_unref_object JsonParser *treefile_parser = NULL; JsonObject *treefile = NULL; /* Owned by parser */ + g_autoptr(RORTreefile) treefile_rs = NULL; + g_auto(GLnxTmpDir) workdir_tmp = { 0, }; if (treefile_path) { - if (!parse_treefile_to_json (NULL, treefile_path, &treefile_parser, error)) + if (!glnx_mkdtempat (AT_FDCWD, "/var/tmp/rpm-ostree.XXXXXX", 0700, &workdir_tmp, error)) + return FALSE; + if (!parse_treefile_to_json (treefile_path, workdir_tmp.fd, NULL, + &treefile_rs, &treefile_parser, error)) return FALSE; JsonNode *treefile_rootval = json_parser_get_root (treefile_parser); diff --git a/src/app/rpmostree-composeutil.c b/src/app/rpmostree-composeutil.c index ab32e9b6..74ce1392 100644 --- a/src/app/rpmostree-composeutil.c +++ b/src/app/rpmostree-composeutil.c @@ -49,7 +49,7 @@ gboolean rpmostree_composeutil_checksum (GBytes *serialized_treefile, HyGoal goal, - GFile *contextdir, + RORTreefile *tf, JsonArray *add_files, char **out_checksum, GError **error) @@ -71,31 +71,19 @@ rpmostree_composeutil_checksum (GBytes *serialized_treefile, guint i, len = json_array_get_length (add_files); for (i = 0; i < len; i++) { - g_autoptr(GFile) srcfile = NULL; - const char *src, *dest; JsonArray *add_el = json_array_get_array_element (add_files, i); - if (!add_el) return glnx_throw (error, "Element in add-files is not an array"); - - src = _rpmostree_jsonutil_array_require_string_element (add_el, 0, error); + const char *src = _rpmostree_jsonutil_array_require_string_element (add_el, 0, error); if (!src) return FALSE; - dest = _rpmostree_jsonutil_array_require_string_element (add_el, 1, error); - if (!dest) - return FALSE; - - srcfile = g_file_resolve_relative_path (contextdir, src); - - if (!_rpmostree_util_update_checksum_from_file (checksum, - AT_FDCWD, - gs_file_get_path_cached (srcfile), - NULL, - error)) - return FALSE; - - g_checksum_update (checksum, (const guint8 *) dest, strlen (dest)); + int src_fd = ror_treefile_get_add_file_fd (tf, src); + g_assert_cmpint (src_fd, !=, -1); + g_autoptr(GBytes) bytes = glnx_fd_readall_bytes (src_fd, NULL, FALSE); + gsize len; + const guint8* buf = g_bytes_get_data (bytes, &len); + g_checksum_update (checksum, (const guint8 *) buf, len); } } @@ -151,34 +139,24 @@ rpmostree_composeutil_legacy_prep_dev (int rootfs_dfd, return TRUE; } + gboolean -rpmostree_composeutil_sanity_checks (JsonObject *treedata, - GFile *contextdir, +rpmostree_composeutil_sanity_checks (RORTreefile *tf, + JsonObject *treefile, GCancellable *cancellable, GError **error) { - /* Check that postprocess-script is executable; https://github.com/projectatomic/rpm-ostree/issues/817 */ - const char *postprocess_script = NULL; + int fd = ror_treefile_get_postprocess_script_fd (tf); + if (fd != -1) + { + /* Check that postprocess-script is executable; https://github.com/projectatomic/rpm-ostree/issues/817 */ + struct stat stbuf; + if (!glnx_fstat (fd, &stbuf, error)) + return glnx_prefix_error (error, "postprocess-script"); - if (!_rpmostree_jsonutil_object_get_optional_string_member (treedata, "postprocess-script", - &postprocess_script, error)) - return FALSE; - - if (!postprocess_script) - return TRUE; - - g_autofree char *src = NULL; - if (g_path_is_absolute (postprocess_script)) - src = g_strdup (postprocess_script); - else - src = g_build_filename (gs_file_get_path_cached (contextdir), postprocess_script, NULL); - - struct stat stbuf; - if (!glnx_fstatat (AT_FDCWD, src, &stbuf, 0, error)) - return glnx_prefix_error (error, "postprocess-script"); - - if ((stbuf.st_mode & S_IXUSR) == 0) - return glnx_throw (error, "postprocess-script (%s) must be executable", postprocess_script); + if ((stbuf.st_mode & S_IXUSR) == 0) + return glnx_throw (error, "postprocess-script must be executable"); + } /* Insert other sanity checks here */ diff --git a/src/app/rpmostree-composeutil.h b/src/app/rpmostree-composeutil.h index 6a8e98cf..e28f6c48 100644 --- a/src/app/rpmostree-composeutil.h +++ b/src/app/rpmostree-composeutil.h @@ -23,13 +23,14 @@ #include #include "rpmostree-core.h" +#include "rpmostree-rust.h" G_BEGIN_DECLS gboolean rpmostree_composeutil_checksum (GBytes *serialized_treefile, HyGoal goal, - GFile *contextdir, + RORTreefile *tf, JsonArray *add_files, char **out_checksum, GError **error); @@ -39,8 +40,8 @@ rpmostree_composeutil_legacy_prep_dev (int rootfs_dfd, GError **error); gboolean -rpmostree_composeutil_sanity_checks (JsonObject *treedata, - GFile *contextdir, +rpmostree_composeutil_sanity_checks (RORTreefile *tf, + JsonObject *treefile, GCancellable *cancellable, GError **error); diff --git a/src/libpriv/rpmostree-postprocess.c b/src/libpriv/rpmostree-postprocess.c index f316b432..a9ec4281 100644 --- a/src/libpriv/rpmostree-postprocess.c +++ b/src/libpriv/rpmostree-postprocess.c @@ -1341,7 +1341,7 @@ rpmostree_rootfs_postprocess_common (int rootfs_fd, */ static gboolean copy_additional_files (int rootfs_dfd, - GFile *context_directory, + RORTreefile *treefile_rs, JsonObject *treefile, GCancellable *cancellable, GError **error) @@ -1356,11 +1356,6 @@ copy_additional_files (int rootfs_dfd, else return TRUE; /* Early return */ - glnx_autofd int context_dfd = -1; - if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (context_directory), TRUE, - &context_dfd, error)) - return FALSE; - /* Reusable dirname buffer */ g_autoptr(GString) dnbuf = g_string_new (""); for (guint i = 0; i < len; i++) @@ -1403,12 +1398,26 @@ copy_additional_files (int rootfs_dfd, if (!glnx_shutil_mkdir_p_at (rootfs_dfd, dn, 0755, cancellable, error)) return FALSE; - /* FIXME: Should probably use GLNX_FILE_COPY_NOXATTRS, but someone - * may be relying on current semantics? + int src_fd = ror_treefile_get_add_file_fd (treefile_rs, src); + g_assert_cmpint (src_fd, !=, -1); + + g_auto(GLnxTmpfile) tmpf = { 0, }; + if (!glnx_open_tmpfile_linkable_at (rootfs_dfd, ".", O_CLOEXEC | O_WRONLY, &tmpf, error)) + return FALSE; + if (glnx_regfile_copy_bytes (src_fd, tmpf.fd, (off_t)-1) < 0) + return glnx_throw_errno_prefix (error, "regfile copy"); + struct stat src_stbuf; + if (!glnx_fstat (src_fd, &src_stbuf, error)) + return FALSE; + if (!glnx_fchmod (tmpf.fd, src_stbuf.st_mode, error)) + return FALSE; + /* Note we used to copy xattrs here, we no longer do. Hopefully + * no one breaks. */ - if (!glnx_file_copy_at (context_dfd, src, NULL, rootfs_dfd, dest, 0, - cancellable, error)) - return glnx_prefix_error (error, "Copying file '%s' into target", src); + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE, + rootfs_dfd, dest, + error)) + return FALSE; } return TRUE; @@ -1464,7 +1473,7 @@ mutate_os_release (const char *contents, */ gboolean rpmostree_treefile_postprocessing (int rootfs_fd, - GFile *context_directory, + RORTreefile *treefile_rs, GBytes *serialized_treefile, JsonObject *treefile, const char *next_version, @@ -1472,7 +1481,7 @@ rpmostree_treefile_postprocessing (int rootfs_fd, GCancellable *cancellable, GError **error) { - g_assert (context_directory); + g_assert (treefile_rs); g_assert (treefile); if (!rename_if_exists (rootfs_fd, "etc", rootfs_fd, "usr/etc", error)) @@ -1720,7 +1729,7 @@ rpmostree_treefile_postprocessing (int rootfs_fd, } /* Copy in additional files before postprocessing */ - if (!copy_additional_files (rootfs_fd, context_directory, treefile, cancellable, error)) + if (!copy_additional_files (rootfs_fd, treefile_rs, treefile, cancellable, error)) return FALSE; if (json_object_has_member (treefile, "postprocess")) @@ -1753,43 +1762,38 @@ rpmostree_treefile_postprocessing (int rootfs_fd, } } - const char *postprocess_script = NULL; - if (!_rpmostree_jsonutil_object_get_optional_string_member (treefile, "postprocess-script", - &postprocess_script, error)) - return FALSE; - if (postprocess_script) + int postprocess_script_fd = ror_treefile_get_postprocess_script_fd (treefile_rs); + if (postprocess_script_fd != -1) { - const char *bn = glnx_basename (postprocess_script); - g_autofree char *src = NULL; - g_autofree char *binpath = NULL; - - if (g_path_is_absolute (postprocess_script)) - src = g_strdup (postprocess_script); - else - src = g_build_filename (gs_file_get_path_cached (context_directory), postprocess_script, NULL); - - binpath = g_strconcat ("/usr/bin/rpmostree-postprocess-", bn, NULL); - /* Clone all the things */ - - /* Note we need to make binpath *not* absolute here */ + const char *binpath = "/usr/bin/rpmostree-treefile-postprocess-script"; const char *target_binpath = binpath + 1; - g_assert_cmpint (*target_binpath, !=, '/'); - if (!glnx_file_copy_at (AT_FDCWD, src, NULL, rootfs_fd, target_binpath, - GLNX_FILE_COPY_NOXATTRS, cancellable, error)) + g_auto(GLnxTmpfile) tmpf = { 0, }; + if (!glnx_open_tmpfile_linkable_at (rootfs_fd, ".", O_CLOEXEC | O_WRONLY, &tmpf, error)) + return FALSE; + if (glnx_regfile_copy_bytes (postprocess_script_fd, tmpf.fd, (off_t)-1) < 0) + return glnx_throw_errno_prefix (error, "regfile copy"); + if (!glnx_fchmod (tmpf.fd, 0755, error)) + return FALSE; + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE, + rootfs_fd, target_binpath, + error)) return FALSE; - g_print ("Executing postprocessing script '%s'\n", bn); + /* Can't have a writable fd open when we go to exec */ + glnx_tmpfile_clear (&tmpf); + + g_print ("Executing postprocessing script\n"); { - char *child_argv[] = { binpath, NULL }; + char *child_argv[] = { (char*)binpath, NULL }; if (!run_bwrap_mutably (rootfs_fd, binpath, child_argv, unified_core_mode, cancellable, error)) - return glnx_prefix_error (error, "While executing postprocessing script '%s'", bn); + return glnx_prefix_error (error, "While executing postprocessing script"); } if (!glnx_unlinkat (rootfs_fd, target_binpath, 0, error)) return FALSE; - g_print ("Finished postprocessing script '%s'\n", bn); + g_print ("Finished postprocessing script\n"); } return TRUE; diff --git a/src/libpriv/rpmostree-postprocess.h b/src/libpriv/rpmostree-postprocess.h index d9661d6d..bf4b6be4 100644 --- a/src/libpriv/rpmostree-postprocess.h +++ b/src/libpriv/rpmostree-postprocess.h @@ -22,6 +22,7 @@ #include #include "rpmostree-json-parsing.h" +#include "rpmostree-rust.h" /* "public" for unit tests */ char * @@ -30,7 +31,7 @@ rpmostree_postprocess_replace_nsswitch (const char *buf, gboolean rpmostree_treefile_postprocessing (int rootfs_fd, - GFile *context_directory, + RORTreefile *treefile_rs, GBytes *serialized_treefile, JsonObject *treefile, const char *next_version,