treefile: Add new add-commit-metadata key

Add support for a new `add-commit-metadata` key in the treefile so that
we can directly specify commit metadata we want to inject from there.

This will be useful in Fedora CoreOS, where we'll have separate
treefiles for each streams, each with stream-specific metadata values
required.

Closes: #1865
Approved by: cgwalters

docs/manual/treefile.md

@@ -182,6 +182,10 @@ It supports the following parameters:
 
    Example: `automatic-version-prefix: "22.0"`
 
+ * `add-commit-metadata`: Map<String, Object>, optional: Metadata to inject as
+   part of composed commits. Keys inserted here can still be overridden at the
+   command line with `--add-metadata-string` or `--add-metadata-from-json`.
+
  * `postprocess-script`: String, optional: Full filesystem path to a script
    that will be executed in the context of the target tree.  The script
    will be copied into the target into `/tmp`, and run as a container

rust/src/treefile.rs

@@ -26,7 +26,7 @@ use openat;
 use serde_derive::{Deserialize, Serialize};
 use serde_json;
 use serde_yaml;
-use std::collections::HashMap;
+use std::collections::{HashMap, BTreeMap};
 use std::io::prelude::*;
 use std::path::Path;
 use std::{collections, fs, io};
@@ -279,6 +279,20 @@ fn merge_vec_field<T>(dest: &mut Option<Vec<T>>, src: &mut Option<Vec<T>>) {
     }
 }
 
+/// Merge a map field similarly to Python's `dict.update()`. In case of
+/// duplicate keys, `dest` wins (`src` is the "included" config).
+fn merge_map_field<T>(
+    dest: &mut Option<BTreeMap<String, T>>,
+    src: &mut Option<BTreeMap<String, T>>)
+{
+    if let Some(mut srcv) = src.take() {
+        if let Some(mut destv) = dest.take() {
+            srcv.append(&mut destv);
+        }
+        *dest = Some(srcv);
+    }
+}
+
 /// Given two configs, merge them.
 fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
     macro_rules! merge_basics {
@@ -291,6 +305,11 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
             $( merge_vec_field(&mut dest.$field, &mut src.$field); )*
         }};
     };
+    macro_rules! merge_maps {
+        ( $($field:ident),* ) => {{
+            $( merge_map_field(&mut dest.$field, &mut src.$field); )*
+        }};
+    };
 
     merge_basics!(
         treeref,
@@ -328,6 +347,9 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
         remove_files,
         remove_from_packages
     );
+    merge_maps!(
+        add_commit_metadata
+    );
 }
 
 /// Merge the treefile externals. There are currently only two keys that
@@ -663,6 +685,12 @@ struct TreeComposeConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(rename = "remove-from-packages")]
     remove_from_packages: Option<Vec<Vec<String>>>,
+    // The BTreeMap here is on purpose; it ensures we always re-serialize in sorted order so that
+    // checksumming is deterministic across runs. (And serde itself uses BTreeMap for child objects
+    // as well).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "add-commit-metadata")]
+    add_commit_metadata: Option<BTreeMap<String, serde_json::Value>>,
 
     #[serde(flatten)]
     legacy_fields: LegacyTreeComposeConfigFields,
@@ -1011,24 +1039,40 @@ rojig:
     #[test]
     fn test_treefile_merge() {
         let basearch = Some(ARCH_X86_64);
-        let mut base_input = io::BufReader::new(VALID_PRELUDE.as_bytes());
-        let mut base = treefile_parse_stream(InputFormat::YAML, &mut base_input, basearch).unwrap();
+        let mut base = append_and_parse(
+            r###"
+add-commit-metadata:
+  my-first-key: "please don't override me"
+  my-second-key: "override me"
+        "###,
+        );
         let mut mid_input = io::BufReader::new(
             r###"
 packages:
   - some layered packages
+add-commit-metadata:
+  my-second-key: "something better"
+  my-third-key: 1000
+  my-fourth-key:
+    nested: table
 "###
             .as_bytes(),
         );
         let mut mid = treefile_parse_stream(InputFormat::YAML, &mut mid_input, basearch).unwrap();
         let mut top_input = io::BufReader::new(ROJIG_YAML.as_bytes());
         let mut top = treefile_parse_stream(InputFormat::YAML, &mut top_input, basearch).unwrap();
+        assert!(top.add_commit_metadata.is_none());
         treefile_merge(&mut mid, &mut base);
         treefile_merge(&mut top, &mut mid);
         let tf = &top;
         assert!(tf.packages.as_ref().unwrap().len() == 8);
         let rojig = tf.rojig.as_ref().unwrap();
         assert!(rojig.name == "exampleos");
+        let data = tf.add_commit_metadata.as_ref().unwrap();
+        assert!(data.get("my-first-key").unwrap().as_str().unwrap() == "please don't override me");
+        assert!(data.get("my-second-key").unwrap().as_str().unwrap() == "something better");
+        assert!(data.get("my-third-key").unwrap().as_i64().unwrap() == 1000);
+        assert!(data.get("my-fourth-key").unwrap().as_object().unwrap().get("nested").unwrap().as_str().unwrap() == "table");
     }
 
     #[test]

src/app/rpmostree-compose-builtin-tree.c

@@ -711,6 +711,18 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
 
   self->metadata = g_hash_table_new_full (g_str_hash, g_str_equal, g_free,
                                           (GDestroyNotify)g_variant_unref);
+
+  /* metadata from the treefile itself has the lowest priority */
+  JsonNode *add_commit_metadata_node =
+    json_object_get_member (self->treefile, "add-commit-metadata");
+  if (add_commit_metadata_node)
+    {
+      if (!rpmostree_composeutil_read_json_metadata (add_commit_metadata_node,
+                                                     self->metadata, error))
+        return FALSE;
+    }
+
+  /* then --add-metadata-from-json */
   if (opt_metadata_json)
     {
       if (!rpmostree_composeutil_read_json_metadata_from_file (opt_metadata_json,
@@ -718,6 +730,7 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
         return FALSE;
     }
 
+  /* and finally --add-metadata-string */
   if (opt_metadata_strings)
     {
       if (!parse_metadata_keyvalue_strings (opt_metadata_strings, self->metadata, error))

tests/compose-tests/libbasic-test.sh

@@ -46,6 +46,10 @@ ostree --repo=${repobuild} show --print-metadata-key exampleos.tests ${treeref}
 assert_file_has_content meta.txt 'smoketested.*e2e'
 ostree --repo=${repobuild} show --print-metadata-key rpmostree.rpmmd-repos ${treeref} > meta.txt
 assert_file_has_content meta.txt 'id.*fedora.*timestamp'
+ostree --repo=${repobuild} show --print-metadata-key foobar  ${treeref} > meta.txt
+assert_file_has_content meta.txt 'bazboo'
+ostree --repo=${repobuild} show --print-metadata-key overrideme  ${treeref} > meta.txt
+assert_file_has_content meta.txt 'new val'
 echo "ok metadata"
 
 ostree --repo=${repobuild} ls -R ${treeref} /usr/lib/ostree-boot > bootls.txt

tests/compose-tests/test-basic-unified.sh

@@ -21,7 +21,8 @@ cat > metadata.json <<EOF
      "rev": "97ec21c614689e533d294cdae464df607b526ab9",
      "src": "https://gitlab.com/exampleos/custom-atomic-host"
   },
-  "exampleos.tests": ["smoketested", "e2e"]
+  "exampleos.tests": ["smoketested", "e2e"],
+  "overrideme": "new val"
 }
 EOF
 runcompose --ex-unified-core --add-metadata-from-json metadata.json

tests/compose-tests/test-basic.sh

@@ -13,7 +13,8 @@ cat > metadata.json <<EOF
      "rev": "97ec21c614689e533d294cdae464df607b526ab9",
      "src": "https://gitlab.com/exampleos/custom-atomic-host"
   },
-  "exampleos.tests": ["smoketested", "e2e"]
+  "exampleos.tests": ["smoketested", "e2e"],
+  "overrideme": "new val"
 }
 EOF
 runcompose --add-metadata-from-json metadata.json

tests/composedata/fedora-base.json

@@ -26,5 +26,9 @@
     "ignore-removed-users": ["root"],
     "ignore-removed-groups": ["root"],
     "check-passwd": { "type": "file", "filename": "passwd" },
-    "check-groups": { "type": "file", "filename": "group" }
+    "check-groups": { "type": "file", "filename": "group" },
+    "add-commit-metadata": {
+        "foobar": "bazboo",
+        "overrideme": "old var"
+    }
 }