repo: Add ostree_repo_write_regfile

This API is push rather than pull, which makes it much more
suitable to use cases like parsing a tar file from external
code.

Now, we have a large mess in this area internally because
the original file writing code was pull based, but static
deltas hit the same problem of wanting a push API, so I added
this special `OstreeRepoBareContent` just for writing regular
files from a push API.

Eventually...I'd like to deprecate the pull based API,
and rework things so that for regular files the push API
is the default, and then `write_content_object()` would
be split up into archive/bare cases.

In this world the `ostree_repo_write_content()` API would
then need to hackily bridge pull to push and it'd be
less efficient.

Anyways for now due to this bifurcation, this API only
works on non-archive repositories, but that's fine for
now because that's what I want for the `ostree-ext-container`
bits.

Makefile-libostree.am

@@ -66,6 +66,8 @@ libostree_1_la_SOURCES = \
 	src/libostree/ostree-checksum-input-stream.h \
 	src/libostree/ostree-chain-input-stream.c \
 	src/libostree/ostree-chain-input-stream.h \
+	src/libostree/ostree-content-writer.c \
+	src/libostree/ostree-content-writer.h \
 	src/libostree/ostree-lzma-common.c \
 	src/libostree/ostree-lzma-common.h \
 	src/libostree/ostree-lzma-compressor.c \

apidoc/ostree-sections.txt

@@ -165,6 +165,12 @@ ostree_check_version
 ostree_commit_sizes_entry_get_type
 </SECTION>
 
+<SECTION>
+<FILE>ostree-content-writer</FILE>
+ostree_content_writer_get_type
+ostree_content_writer_finish
+</SECTION>
+
 <SECTION>
 <FILE>ostree-deployment</FILE>
 OstreeDeployment
@@ -355,6 +361,7 @@ ostree_repo_write_metadata
 ostree_repo_write_metadata_async
 ostree_repo_write_metadata_finish
 ostree_repo_write_content
+ostree_repo_write_regfile
 ostree_repo_write_regfile_inline
 ostree_repo_write_symlink
 ostree_repo_write_metadata_trusted

src/libostree/libostree-devel.sym

@@ -26,6 +26,9 @@ LIBOSTREE_2021.2 {
 global:
   ostree_repo_write_regfile_inline;
   ostree_repo_write_symlink;
+  ostree_repo_write_regfile;
+  ostree_content_writer_get_type;
+  ostree_content_writer_finish;
 } LIBOSTREE_2021.1;
 
 /* Stub section for the stable release *after* this development one; don't

src/libostree/ostree-content-writer.c

@@ -0,0 +1,144 @@
+/* 
+ * SPDX-License-Identifier: LGPL-2.0+
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General
+ * Public License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place, Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ostree-content-writer.h"
+#include "ostree-repo-private.h"
+#include "ostree-autocleanups.h"
+
+struct _OstreeContentWriter
+{
+  GOutputStream parent_instance;
+
+  OstreeRepo *repo;
+  OstreeRepoBareContent output;
+};
+
+G_DEFINE_TYPE (OstreeContentWriter, ostree_content_writer, G_TYPE_OUTPUT_STREAM)
+
+static void     ostree_content_writer_finalize     (GObject *object);
+static gssize   ostree_content_writer_write         (GOutputStream         *stream,
+                                                     const void                 *buffer,
+                                                     gsize                 count,
+                                                     GCancellable         *cancellable,
+                                                     GError              **error);
+static gboolean ostree_content_writer_close        (GOutputStream         *stream,
+                                                    GCancellable         *cancellable,
+                                                    GError              **error);
+
+static void
+ostree_content_writer_class_init (OstreeContentWriterClass *klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GOutputStreamClass *stream_class = G_OUTPUT_STREAM_CLASS (klass);
+  
+  gobject_class->finalize     = ostree_content_writer_finalize;
+
+  stream_class->write_fn = ostree_content_writer_write;
+  stream_class->close_fn = ostree_content_writer_close;
+}
+
+static void
+ostree_content_writer_finalize (GObject *object)
+{
+  OstreeContentWriter *stream;
+
+  stream = (OstreeContentWriter*)(object);
+
+  g_clear_object (&stream->repo);
+  _ostree_repo_bare_content_cleanup (&stream->output);
+
+  G_OBJECT_CLASS (ostree_content_writer_parent_class)->finalize (object);
+}
+
+static void
+ostree_content_writer_init (OstreeContentWriter *self)
+{
+  self->output.initialized = FALSE;
+ }
+
+OstreeContentWriter *
+_ostree_content_writer_new (OstreeRepo           *repo,
+                           const char            *checksum,
+                           guint                  uid,
+                           guint                  gid,
+                           guint                  mode,
+                           guint64                content_len,
+                           GVariant              *xattrs,
+                           GError               **error)
+{
+  g_autoptr(OstreeContentWriter) stream = g_object_new (OSTREE_TYPE_CONTENT_WRITER, NULL);
+  stream->repo = g_object_ref (repo);
+  if (!_ostree_repo_bare_content_open (stream->repo, checksum, content_len, uid, gid, mode, xattrs,
+                                       &stream->output, NULL, error))
+    return NULL;
+  return g_steal_pointer (&stream);
+}
+
+static gssize
+ostree_content_writer_write (GOutputStream  *stream,
+                             const void    *buffer,
+                             gsize          count,
+                             GCancellable  *cancellable,
+                             GError       **error)
+{
+  OstreeContentWriter *self = (OstreeContentWriter*) stream;
+
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return -1;
+
+  if (!_ostree_repo_bare_content_write (self->repo, &self->output,
+                                        buffer, count, cancellable, error))
+    return -1;
+  return count;
+}
+
+static gboolean
+ostree_content_writer_close (GOutputStream        *stream,
+                             GCancellable         *cancellable,
+                             GError              **error)
+{
+  /* We don't expect people to invoke close() - they need to call finish()
+   * to get the checksum.  We'll clean up in finalize anyways if need be.
+   */
+  return TRUE;
+}
+
+/**
+ * ostree_content_writer_finish:
+ * @self: Writer
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Complete the object write and return the checksum.
+ * Returns: (transfer full): Checksum, or %NULL on error
+ */
+char *
+ostree_content_writer_finish (OstreeContentWriter  *self,
+                              GCancellable         *cancellable,
+                              GError              **error)
+{
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  if (!_ostree_repo_bare_content_commit (self->repo, &self->output, actual_checksum,
+                                         sizeof (actual_checksum), cancellable, error))
+    return NULL;
+
+  return g_strdup (actual_checksum);
+}

src/libostree/ostree-content-writer.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2021 Red Hat, Inc.
+ *
+ * SPDX-License-Identifier: LGPL-2.0+
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#include <gio/gio.h>
+
+G_BEGIN_DECLS
+
+#define OSTREE_TYPE_CONTENT_WRITER (ostree_content_writer_get_type ())
+_OSTREE_PUBLIC G_DECLARE_FINAL_TYPE (OstreeContentWriter, ostree_content_writer, OSTREE, CONTENT_WRITER, GOutputStream)
+
+_OSTREE_PUBLIC
+char * ostree_content_writer_finish (OstreeContentWriter  *self,
+                                     GCancellable         *cancellable,
+                                     GError              **error);
+
+G_END_DECLS

src/libostree/ostree-repo-commit.c

@@ -2855,6 +2855,39 @@ ostree_repo_write_symlink (OstreeRepo       *self,
   return ostree_checksum_from_bytes (csum);
 }
 
+/**
+ * ostree_repo_write_regfile:
+ * @self: Repo,
+ * @expected_checksum: (allow-none): Expected checksum (SHA-256 hex string)
+ * @uid: user id
+ * @gid: group id
+ * @mode: Unix file mode
+ * @content_len: Expected content length
+ * @xattrs: (allow-none): Extended attributes (GVariant type `(ayay)`)
+ * @error: Error
+ * 
+ * Create an `OstreeContentWriter` that allows streaming output into
+ * the repository.
+ *
+ * Returns: (transfer full): A new writer, or %NULL on error
+ * Since: 2021.2
+ */
+OstreeContentWriter *    
+ostree_repo_write_regfile (OstreeRepo       *self,
+                           const char       *expected_checksum,
+                           guint32           uid,
+                           guint32           gid,
+                           guint32           mode,
+                           guint64           content_len,
+                           GVariant         *xattrs,
+                           GError          **error)
+{
+  if (self->mode == OSTREE_REPO_MODE_ARCHIVE)
+    return glnx_null_throw (error, "Cannot currently use ostree_repo_write_regfile() on an archive mode repository");
+
+  return _ostree_content_writer_new (self, expected_checksum, uid, gid, mode, content_len, xattrs, error);
+}
+
 typedef struct {
   OstreeRepo *repo;
   char *expected_checksum;

src/libostree/ostree-repo-private.h

@@ -462,6 +462,16 @@ _ostree_repo_bare_content_commit (OstreeRepo                 *self,
                                   GCancellable               *cancellable,
                                   GError                    **error);
 
+OstreeContentWriter *
+_ostree_content_writer_new (OstreeRepo            *repo,
+                           const char            *checksum,
+                           guint                  uid,
+                           guint                  gid,
+                           guint                  mode,
+                           guint64                content_len,
+                           GVariant              *xattrs,
+                           GError               **error);
+
 gboolean
 _ostree_repo_load_file_bare (OstreeRepo         *self,
                              const char         *checksum,

src/libostree/ostree-repo.h

@@ -435,6 +435,16 @@ char *      ostree_repo_write_regfile_inline (OstreeRepo       *self,
                                                 GCancellable     *cancellable,
                                                 GError          **error);
 
+_OSTREE_PUBLIC
+OstreeContentWriter *    ostree_repo_write_regfile (OstreeRepo       *self,
+                                                    const char       *expected_checksum,
+                                                    guint32           uid,
+                                                    guint32           gid,
+                                                    guint32           mode,
+                                                    guint64           content_len,
+                                                    GVariant         *xattrs,
+                                                    GError          **error);
+             
 _OSTREE_PUBLIC
 char *      ostree_repo_write_symlink (OstreeRepo       *self,
                                        const char       *expected_checksum,

src/libostree/ostree-types.h

@@ -38,6 +38,7 @@ typedef struct OstreeSysroot OstreeSysroot;
 typedef struct OstreeSysrootUpgrader OstreeSysrootUpgrader;
 typedef struct OstreeMutableTree OstreeMutableTree;
 typedef struct OstreeRepoFile OstreeRepoFile;
+typedef struct _OstreeContentWriter OstreeContentWriter;
 typedef struct OstreeRemote OstreeRemote;
 
 G_END_DECLS

tests/test-core.js

@@ -27,6 +27,22 @@ function assertEquals(a, b) {
 	throw new Error("assertion failed " + JSON.stringify(a) + " == " + JSON.stringify(b));
 }
 
+function assertThrows(s, f) {
+  let success = false;
+  try {
+    f();
+    success = true;
+  } catch(e) {
+    let msg = e.toString();
+    if (msg.indexOf(s) == -1) {
+      throw new Error("Error message didn't match '" + s + "': " + msg)    
+    }
+  }
+  if (success) {
+    throw new Error("Function was expected to throw, but didn't")
+  }
+}
+
 print('1..1')
 
 let testDataDir = Gio.File.new_for_path('test-data');
@@ -51,18 +67,14 @@ print("commit => " + commit);
 
 // Test direct write APIs
 let inline_content = "default 0.0.0.0\nloopback 127.0.0.0\nlink-local 169.254.0.0\n";
+let networks_checksum = "8aaa9dc13a0c5839fe4a277756798c609c53fac6fa2290314ecfef9041065873";
 let regfile_mode = 33188; // 0o100000 | 0o644 (but in decimal so old gjs works)
 let inline_checksum = repo.write_regfile_inline(null, 0, 0, regfile_mode, null, inline_content, null);
 assertEquals(inline_checksum, "8aaa9dc13a0c5839fe4a277756798c609c53fac6fa2290314ecfef9041065873");
-let written = false;
-try {
+assertThrows("Corrupted file object", function() {
     // Changed an a to b from above to make the checksum not match
     repo.write_regfile_inline("8baa9dc13a0c5839fe4a277756798c609c53fac6fa2290314ecfef9041065873", 0, 0, regfile_mode, null, inline_content, null);
-    written = true;
-} catch (e) {
-}
-if (written)
-  throw new Error("Wrote invalid checksum");
+});
 
 repo.commit_transaction(null, null);
 
@@ -85,4 +97,21 @@ repo.commit_transaction(null, null);
 [,readCommit] = repo.resolve_rev('someref', true);
 assertEquals(readCommit, null);
 
+// Test direct write API for regular files
+let clen = inline_content.length;
+assertThrows("Cannot currently use", function() {  
+  let w = repo.write_regfile(null, 0, 0, regfile_mode, clen, null);
+});
+
+let bareRepoPath = Gio.File.new_for_path('repo');
+let repo_bareu = OSTree.Repo.new(Gio.File.new_for_path('repo-bare'));
+repo_bareu.create(OSTree.RepoMode.BARE_USER_ONLY, null);
+let w = repo_bareu.write_regfile(null, 0, 0, regfile_mode, clen, null);
+// Test multiple write() calls
+w.write(inline_content.slice(0, 4), null)
+w.write(inline_content.slice(4, 10), null)
+w.write(inline_content.slice(10), null)
+let actual_checksum = w.finish(null)
+assertEquals(actual_checksum, networks_checksum)
+
 print("ok test-core");

tests/test-repo.c

@@ -241,7 +241,6 @@ test_write_regfile_api (Fixture *fixture,
   g_clear_pointer (&xattrs, g_variant_unref);
   g_variant_builder_init (&xattrs_builder, (GVariantType*)"a(ayay)");
   g_variant_builder_add (&xattrs_builder, "(^ay^ay)", "security.selinux", "system_u:object_r:bin_t:s0");
-  g_clear_pointer (&xattrs, g_variant_unref);
   xattrs = g_variant_ref_sink (g_variant_builder_end (&xattrs_builder));
 
   g_clear_pointer (&checksum, g_free);