Add public API for fsck, use it before loading metadata

A while ago I did `truncate -s 0 /path/to/repo/00/123.commit`, and expected a
checksum error, but I actually got a validation error due to us loading the
commit into a variant and trying to parse out the parent checksum, etc.

I first started by changing the `load_and_fsck_one_object()` function to
checksum before loading, but the problem is that we do a traverse of all objects
first. Fixing this is going to require an `OSTREE_REPO_COMMIT_TRAVER_FLAG_FSCK`
or something.

In the meantime at least though, let's add a public API to fsck a single object
which *does* checksum cleanly before parsing the object, and change the `fsck`
command to use it.

We then change the fsck binary to do this while iterating over the refs
and finding the commit object.  This way we'll at least get a checksum
first for commit objects, even if not dirtree/dirmeta.

Closes: #1364
Approved by: jlebon

apidoc/ostree-sections.txt

@@ -348,6 +348,7 @@ ostree_repo_import_object_from_with_trust
 ostree_repo_import_archive_to_mtree
 ostree_repo_export_tree_to_archive
 ostree_repo_delete_object
+ostree_repo_fsck_object
 OstreeRepoCommitFilterResult
 OstreeRepoCommitFilter
 OstreeRepoCommitModifier

src/libostree/libostree-devel.sym

@@ -20,6 +20,7 @@
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
 
 LIBOSTREE_2017.15 {
+  ostree_repo_fsck_object;
 } LIBOSTREE_2017.14;
 
 /* Stub section for the stable release *after* this development one; don't

src/libostree/ostree-repo.c

@@ -3914,6 +3914,115 @@ ostree_repo_delete_object (OstreeRepo           *self,
   return TRUE;
 }
 
+static gboolean
+fsck_metadata_object (OstreeRepo           *self,
+                      OstreeObjectType      objtype,
+                      const char           *sha256,
+                      GCancellable         *cancellable,
+                      GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck ", sha256, ".", ostree_object_type_to_string (objtype));
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GVariant) metadata = NULL;
+  if (!load_metadata_internal (self, objtype, sha256, TRUE,
+                               &metadata, NULL, NULL, NULL,
+                               cancellable, error))
+    return FALSE;
+
+  g_auto(OtChecksum) hasher = { 0, };
+  ot_checksum_init (&hasher);
+  ot_checksum_update (&hasher, g_variant_get_data (metadata), g_variant_get_size (metadata));
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ot_checksum_get_hexdigest (&hasher, actual_checksum, sizeof (actual_checksum));
+  if (!_ostree_compare_object_checksum (objtype, sha256, actual_checksum, error))
+    return FALSE;
+
+  switch (objtype)
+    {
+    case OSTREE_OBJECT_TYPE_COMMIT:
+      if (!ostree_validate_structureof_commit (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_TREE:
+      if (!ostree_validate_structureof_dirtree (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_META:
+      if (!ostree_validate_structureof_dirmeta (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT:
+    case OSTREE_OBJECT_TYPE_COMMIT_META:
+      /* TODO */
+      break;
+    case OSTREE_OBJECT_TYPE_FILE:
+      g_assert_not_reached ();
+      break;
+    }
+
+  return TRUE;
+}
+
+static gboolean
+fsck_content_object (OstreeRepo           *self,
+                     const char           *sha256,
+                     GCancellable         *cancellable,
+                     GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck content object ", sha256);
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GInputStream) input = NULL;
+  g_autoptr(GFileInfo) file_info = NULL;
+  g_autoptr(GVariant) xattrs = NULL;
+
+  if (!ostree_repo_load_file (self, sha256, &input, &file_info, &xattrs,
+                              cancellable, error))
+    return FALSE;
+
+  /* TODO more consistency checks here */
+  const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+  if (!ostree_validate_structureof_file_mode (mode, error))
+    return FALSE;
+
+  g_autofree guchar *computed_csum = NULL;
+  if (!ostree_checksum_file_from_input (file_info, xattrs, input,
+                                        OSTREE_OBJECT_TYPE_FILE, &computed_csum,
+                                        cancellable, error))
+    return FALSE;
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ostree_checksum_inplace_from_bytes (computed_csum, actual_checksum);
+  return _ostree_compare_object_checksum (OSTREE_OBJECT_TYPE_FILE, sha256, actual_checksum, error);
+}
+
+/**
+ * ostree_repo_fsck_object:
+ * @self: Repo
+ * @objtype: Object type
+ * @sha256: Checksum
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Verify consistency of the object; this performs checks only relevant to the
+ * immediate object itself, such as checksumming. This API call will not itself
+ * traverse metadata objects for example.
+ *
+ * Since: 2017.15
+ */
+gboolean
+ostree_repo_fsck_object (OstreeRepo           *self,
+                         OstreeObjectType      objtype,
+                         const char           *sha256,
+                         GCancellable         *cancellable,
+                         GError              **error)
+{
+  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+    return fsck_metadata_object (self, objtype, sha256, cancellable, error);
+  else
+    return fsck_content_object (self, sha256, cancellable, error);
+}
+
 /**
  * ostree_repo_import_object_from:
  * @self: Destination repo

src/libostree/ostree-repo.h

@@ -654,6 +654,13 @@ gboolean      ostree_repo_delete_object (OstreeRepo           *self,
                                          GCancellable         *cancellable,
                                          GError              **error);
 
+_OSTREE_PUBLIC
+gboolean      ostree_repo_fsck_object (OstreeRepo           *self,
+                                       OstreeObjectType      objtype,
+                                       const char           *sha256,
+                                       GCancellable         *cancellable,
+                                       GError              **error);
+
 /** 
  * OstreeRepoCommitFilterResult:
  * @OSTREE_REPO_COMMIT_FILTER_ALLOW: Do commit this object

src/ostree/ot-builtin-fsck.c

@@ -44,118 +44,36 @@ static GOptionEntry options[] = {
 };
 
 static gboolean
-load_and_fsck_one_object (OstreeRepo            *repo,
+fsck_one_object (OstreeRepo            *repo,
                  const char            *checksum,
                  OstreeObjectType       objtype,
                  gboolean              *out_found_corruption,
                  GCancellable          *cancellable,
                  GError               **error)
 {
-  gboolean missing = FALSE;
-  g_autoptr(GVariant) metadata = NULL;
-  g_autoptr(GInputStream) input = NULL;
-  g_autoptr(GFileInfo) file_info = NULL;
-  g_autoptr(GVariant) xattrs = NULL;
   g_autoptr(GError) temp_error = NULL;
-
+  if (!ostree_repo_fsck_object (repo, objtype, checksum, cancellable, &temp_error))
-  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
-    {
-      if (!ostree_repo_load_variant (repo, objtype,
-                                     checksum, &metadata, &temp_error))
     {
       if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
           g_clear_error (&temp_error);
           g_printerr ("Object missing: %s.%s\n", checksum,
                       ostree_object_type_to_string (objtype));
-              missing = TRUE;
-            }
-          else
-            {
-              g_propagate_error (error, g_steal_pointer (&temp_error));
-              return glnx_prefix_error (error, "Loading metadata object %s", checksum);
-            }
-        }
-      else
-        {
-          if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
-            {
-              if (!ostree_validate_structureof_commit (metadata, error))
-                return glnx_prefix_error (error, "While validating commit metadata '%s'", checksum);
-            }
-          else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE)
-            {
-              if (!ostree_validate_structureof_dirtree (metadata, error))
-                return glnx_prefix_error (error, "While validating directory tree '%s'", checksum);
-            }
-          else if (objtype == OSTREE_OBJECT_TYPE_DIR_META)
-            {
-              if (!ostree_validate_structureof_dirmeta (metadata, error))
-                return glnx_prefix_error (error, "While validating directory metadata '%s'", checksum);
-            }
-
-          input = g_memory_input_stream_new_from_data (g_variant_get_data (metadata),
-                                                       g_variant_get_size (metadata),
-                                                       NULL);
-
-        }
-    }
-  else
-    {
-      guint32 mode;
-      g_assert (objtype == OSTREE_OBJECT_TYPE_FILE);
-      if (!ostree_repo_load_file (repo, checksum, &input, &file_info,
-                                  &xattrs, cancellable, &temp_error))
-        {
-          if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-            {
-              g_clear_error (&temp_error);
-              g_printerr ("Object missing: %s.%s\n", checksum,
-                          ostree_object_type_to_string (objtype));
-              missing = TRUE;
-            }
-          else
-            {
-              g_propagate_error (error, g_steal_pointer (&temp_error));
-              return glnx_prefix_error (error, "Loading file object %s", checksum);
-            }
-        }
-      else
-        {
-          mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
-          if (!ostree_validate_structureof_file_mode (mode, error))
-            return glnx_prefix_error (error, "While validating file '%s'", checksum);
-        }
-    }
-
-  if (missing)
-    {
           *out_found_corruption = TRUE;
         }
       else
         {
-      g_autofree guchar *computed_csum = NULL;
-      g_autofree char *tmp_checksum = NULL;
-
-      if (!ostree_checksum_file_from_input (file_info, xattrs, input,
-                                            objtype, &computed_csum,
-                                            cancellable, error))
-        return FALSE;
-
-      tmp_checksum = ostree_checksum_from_bytes (computed_csum);
-      if (strcmp (checksum, tmp_checksum) != 0)
-        {
-          g_autofree char *msg = g_strdup_printf ("corrupted object %s.%s; actual checksum: %s",
-                                               checksum, ostree_object_type_to_string (objtype),
-                                               tmp_checksum);
           if (opt_delete)
             {
-              g_printerr ("%s\n", msg);
+              g_printerr ("%s\n", temp_error->message);
               (void) ostree_repo_delete_object (repo, objtype, checksum, cancellable, NULL);
               *out_found_corruption = TRUE;
             }
           else
-            return glnx_throw (error, "%s", msg);
+            {
+              g_propagate_error (error, g_steal_pointer (&temp_error));
+              return FALSE;
+            }
         }
     }
 
@@ -201,7 +119,7 @@ fsck_reachable_objects_from_commits (OstreeRepo            *repo,
 
       ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
 
-      if (!load_and_fsck_one_object (repo, checksum, objtype, out_found_corruption,
+      if (!fsck_one_object (repo, checksum, objtype, out_found_corruption,
                             cancellable, error))
         return FALSE;
 
@@ -239,6 +157,12 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation,
     {
       const char *refname = key;
       const char *checksum = value;
+
+      if (!fsck_one_object (repo, checksum, OSTREE_OBJECT_TYPE_COMMIT,
+                            &found_corruption,
+                            cancellable, error))
+        return FALSE;
+
       g_autoptr(GVariant) commit = NULL;
       if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT,
                                      checksum, &commit, error))

tests/pull-test.sh

@@ -183,7 +183,7 @@ if ! skip_one_without_user_xattrs; then
         if ${CMD_PREFIX} ostree --repo=cacherepo fsck 2>err.txt; then
             fatal "corrupt repo fsck?"
         fi
-        assert_file_has_content err.txt "corrupted.*${checksum}"
+        assert_file_has_content err.txt "Corrupted.*${checksum}"
         rm ostree-srv/corruptrepo -rf
         ostree_repo_init ostree-srv/corruptrepo --mode=archive
         ${CMD_PREFIX} ostree --repo=ostree-srv/corruptrepo pull-local cacherepo main

tests/test-corruption.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+# Copyright (C) 2011,2017 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..3"
+echo "1..4"
 
 . $(dirname $0)/libtest.sh
 
@@ -35,6 +35,18 @@ $OSTREE fsck -q
 
 echo "ok chmod"
 
+cd ${test_tmpdir}
+rm repo files -rf
+setup_test_repository "bare"
+rev=$($OSTREE rev-parse test2)
+echo -n > repo/objects/${rev:0:2}/${rev:2}.commit
+if $OSTREE fsck -q 2>err.txt; then
+    fatal "fsck unexpectedly succeeded"
+fi
+assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected"
+
+echo "ok metadata checksum"
+
 cd ${test_tmpdir}
 rm repo files -rf
 setup_test_repository "bare"

tests/test-pull-corruption.sh

@@ -76,7 +76,7 @@ if ! skip_one_without_user_xattrs; then
     if ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo fsck 2>err.txt; then
         assert_not_reached "fsck with corrupted commit worked?"
     fi
-    assert_file_has_content err.txt "corrupted object ${corruptrev}\.commit"
+    assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected='${corruptrev}' actual='${rev}'"
 
     # Do a pull-local; this should succeed since we don't verify checksums
     # for local repos by default.