oxidation: Add implementation of bupsplit in Rust

This is an initial drop of "oxidation", or adding implementation
of components in Rust.  The bupsplit code is a good target - no
dependencies, just computation.

Translation into Rust had a few twists -

 - The C code relies a lot on overflowing unsigned ints, and
   also on the C promotion rules for e.g. `uint8_t -> int32_t`
 - There were some odd loops that I introduced bugs in while
   translating...in particular, the function always returns `len`,
   but I mistakenly translated to `len+1`, resulting in an OOB
   read on the C side, which was hard to debug.

On the plus side, an off-by-one array indexing in the Rust code paniced nicely.

In practice, we'll need a lot more build infrastructure to make this work, such
as using `cargo vendor` when producing build artifacts for example. Also, Cargo
is yet another thing we need to cache.

Where do we go with this? Well, I think we should merge this, it's not a lot of
code. We can just have it be an alternative CI target. Should we do a lot more
right now? Probably not immediately, but I find the medium/long term prospects
pretty exciting!

Closes: #656
Approved by: jlebon

.redhat-ci.yml

@@ -46,3 +46,30 @@ env:
 
 tests:
 artifacts:
+
+
+---
+
+inherit: true
+
+context: f25-rust
+
+packages:
+  - cargo
+
+build:
+    config-opts: >
+      --prefix=/usr
+      --libdir=/usr/lib64
+      --enable-installed-tests
+      --enable-gtk-doc
+      --enable-rust
+
+env:
+    CC: 'gcc'
+
+tests:
+    - make check TESTS=tests/test-rollsum
+
+artifacts:
+  - test-suite.log

Makefile-libostree.am

@@ -19,7 +19,19 @@
 
 include Makefile-libostree-defines.am
 
-noinst_LTLIBRARIES += libostree-kernel-args.la libbupsplit.la
+noinst_LTLIBRARIES += libostree-kernel-args.la
+
+
+if ENABLE_RUST
+bupsplitpath = @abs_top_builddir@/target/@RUST_TARGET_SUBDIR@/libbupsplit_rs.a
+.PHONY: $(bupsplitpath)
+$(bupsplitpath): Makefile rust/src/bupsplit.rs
+	cd $(top_srcdir)/rust && CARGO_TARGET_DIR=@abs_top_builddir@/target cargo build --verbose $(CARGO_RELEASE_ARGS)
+else
+bupsplitpath = libbupsplit.la
+noinst_LTLIBRARIES += libbupsplit.la
+libbupsplit_la_SOURCES = src/libostree/bupsplit.h src/libostree/bupsplit.c
+endif # ENABLE_RUST
 
 libostree_kernel_args_la_SOURCES = \
 	src/libostree/ostree-kernel-args.h \
@@ -56,11 +68,6 @@ BUILT_SOURCES += $(nodist_libostree_1_la_SOURCES)
 
 CLEANFILES += $(BUILT_SOURCES)
 
-libbupsplit_la_SOURCES = \
-	src/libostree/bupsplit.h \
-	src/libostree/bupsplit.c \
-	$(NULL)
-
 libostree_1_la_SOURCES = \
 	src/libostree/ostree-async-progress.c \
 	src/libostree/ostree-cmdprivate.h \
@@ -142,7 +149,8 @@ libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$(
 	$(OT_INTERNAL_GIO_UNIX_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS) $(OT_DEP_LZMA_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) \
 	-fvisibility=hidden '-D_OSTREE_PUBLIC=__attribute__((visibility("default"))) extern'
 libostree_1_la_LDFLAGS = -version-number 1:0:0 -Bsymbolic-functions -Wl,--version-script=$(top_srcdir)/src/libostree/libostree.sym
-libostree_1_la_LIBADD = libotutil.la libbupsplit.la libglnx.la libbsdiff.la libostree-kernel-args.la $(OT_INTERNAL_GIO_UNIX_LIBS) $(OT_INTERNAL_GPGME_LIBS) $(OT_DEP_LZMA_LIBS) $(OT_DEP_ZLIB_LIBS)
+libostree_1_la_LIBADD = libotutil.la libglnx.la libbsdiff.la libostree-kernel-args.la $(OT_INTERNAL_GIO_UNIX_LIBS) $(OT_INTERNAL_GPGME_LIBS) $(OT_DEP_LZMA_LIBS) $(OT_DEP_ZLIB_LIBS)
+libostree_1_la_LIBADD += $(bupsplitpath)
 EXTRA_libostree_1_la_DEPENDENCIES = $(top_srcdir)/src/libostree/libostree.sym
 
 EXTRA_DIST += src/libostree/libostree.sym

Makefile-tests.am

@@ -200,11 +200,11 @@ TESTS_LDADD = $(common_tests_ldadd) libostreetest.la
 
 tests_test_rollsum_cli_SOURCES = src/libostree/ostree-rollsum.c tests/test-rollsum-cli.c
 tests_test_rollsum_cli_CFLAGS = $(TESTS_CFLAGS) $(OT_DEP_ZLIB_CFLAGS)
-tests_test_rollsum_cli_LDADD = libbupsplit.la $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS)
+tests_test_rollsum_cli_LDADD = $(bupsplitpath) $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS)
 
 tests_test_rollsum_SOURCES = src/libostree/ostree-rollsum.c tests/test-rollsum.c
 tests_test_rollsum_CFLAGS = $(TESTS_CFLAGS) $(OT_DEP_ZLIB_CFLAGS)
-tests_test_rollsum_LDADD = libbupsplit.la $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS)
+tests_test_rollsum_LDADD = $(bupsplitpath) $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS)
 
 tests_test_mutable_tree_CFLAGS = $(TESTS_CFLAGS)
 tests_test_mutable_tree_LDADD = $(TESTS_LDADD)

Makefile.am

@@ -60,6 +60,21 @@ GIRS =
 TYPELIBS = $(GIRS:.gir=.typelib)
 endif
 
+# These bits based on gnome:librsvg/Makefile.am
+if ENABLE_RUST
+if RUST_DEBUG
+CARGO_RELEASE_ARGS=
+else
+CARGO_RELEASE_ARGS=--release
+endif
+
+check-local:
+	cd $(srcdir)/rust && CARGO_TARGET_DIR=$(abs_top_builddir)/target cargo test
+
+clean-local:
+	cd $(srcdir)/rust && CARGO_TARGET_DIR=$(abs_top_builddir)/target cargo clean
+endif # end ENABLE_RUST
+
 libglnx_srcpath := $(srcdir)/libglnx
 libglnx_cflags := $(OT_DEP_GIO_UNIX_CFLAGS) "-I$(libglnx_srcpath)"
 libglnx_libs := $(OT_DEP_GIO_UNIX_LIBS)

configure.ac

@@ -166,6 +166,39 @@ AS_IF([test "$enable_man" != no], [
 ])
 AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no)
 
+AC_ARG_ENABLE(rust,
+  [AS_HELP_STRING([--enable-rust],
+  [Compile Rust code instead of C [default=no]])],,
+   [enable_rust=no; rust_debug_release=no])
+
+AS_IF([test "$enable_rust" = yes], [
+   AC_PATH_PROG([cargo], [cargo])
+   AS_IF([test -z "$cargo"], [AC_MSG_ERROR([cargo is required for --enable-rust])])
+   AC_PATH_PROG([rustc], [rustc])
+   AS_IF([test -z "$rustc"], [AC_MSG_ERROR([rustc is required for --enable-rust])])
+
+   dnl These bits based on gnome:librsvg/configure.ac
+
+   dnl By default, we build in public release mode.
+   AC_ARG_ENABLE(rust-debug,
+      AC_HELP_STRING([--enable-rust-debug],
+       [Build Rust code with debugging information [default=no]]),
+       [rust_debug_release=$enableval],
+       [rust_debug_release=release])
+
+   AC_MSG_CHECKING(whether to build Rust code with debugging information)
+   if test "x$rust_debug_release" = "xyes" ; then
+      rust_debug_release=debug
+      AC_MSG_RESULT(yes)
+   else
+      AC_MSG_RESULT(no)
+   fi
+   RUST_TARGET_SUBDIR=${rust_debug_release}
+   AC_SUBST([RUST_TARGET_SUBDIR])
+])
+AM_CONDITIONAL(RUST_DEBUG, [test "x$rust_debug_release" = "xdebug"])
+AM_CONDITIONAL(ENABLE_RUST, [test "$enable_rust" != no])
+
 AC_ARG_WITH(libarchive,
 	    AS_HELP_STRING([--without-libarchive], [Do not use libarchive]),
 	    :, with_libarchive=maybe)
@@ -339,6 +372,7 @@ echo "
 
 
     introspection:                                $found_introspection
+    Rust (internal oxidation):                    $rust_debug_release
     rofiles-fuse:                                 $enable_rofiles_fuse
     libsoup (retrieve remote HTTP repositories):  $with_soup
     libsoup TLS client certs:                     $have_libsoup_client_certs

rust/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "bupsplit"
+version = "0.0.1"
+authors = ["Colin Walters <walters@verbum.org>"]
+
+[dependencies]
+libc = "0.2"
+
+[lib]
+name = "bupsplit_rs"
+path = "src/bupsplit.rs"
+crate-type = ["staticlib"]
+
+[profile.release]
+panic = "abort"
+lto = true

rust/src/bupsplit.rs

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2017 Colin Walters <walters@verbum.org>
+ * Based on original bupsplit.c:
+ * Copyright 2011 Avery Pennarun. All rights reserved.
+ *
+ * (This license applies to bupsplit.c and bupsplit.h only.)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *    1. Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AVERY PENNARUN ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+extern crate libc;
+
+use std::slice;
+
+// According to librsync/rollsum.h:
+// "We should make this something other than zero to improve the
+// checksum algorithm: tridge suggests a prime number."
+// apenwarr: I unscientifically tried 0 and 7919, and they both ended up
+// slightly worse than the librsync value of 31 for my arbitrary test data.
+const ROLLSUM_CHAR_OFFSET: u32 = 31;
+
+// Previously in the header file
+const BUP_BLOBBITS: u32= 13;
+const BUP_BLOBSIZE: u32 = (1<<BUP_BLOBBITS);
+const BUP_WINDOWBITS: u32 = 7;
+const BUP_WINDOWSIZE: u32 = (1<<(BUP_WINDOWBITS-1));
+
+struct Rollsum {
+    s1: u32,
+    s2: u32,
+    window: [u8; BUP_WINDOWSIZE as usize],
+    wofs: i32,
+}
+
+impl Rollsum {
+    pub fn new() -> Rollsum {
+        Rollsum { s1: BUP_WINDOWSIZE * ROLLSUM_CHAR_OFFSET,
+                  s2: BUP_WINDOWSIZE * (BUP_WINDOWSIZE-1) * ROLLSUM_CHAR_OFFSET,
+                  window: [0; 64],
+                  wofs: 0
+        }
+    }
+
+    // These formulas are based on rollsum.h in the librsync project.
+    pub fn add(&mut self, drop: u8, add: u8) -> () {
+        let drop_expanded = drop as u32;
+        let add_expanded = add as u32;
+        self.s1 = self.s1.wrapping_add(add_expanded.wrapping_sub(drop_expanded));
+        self.s2 = self.s2.wrapping_add(self.s1.wrapping_sub(BUP_WINDOWSIZE * (drop_expanded + ROLLSUM_CHAR_OFFSET)));
+    }
+
+    pub fn roll(&mut self, ch: u8) -> () {
+        let wofs = self.wofs as usize;
+        let dval = self.window[wofs];
+        self.add(dval, ch);
+        self.window[wofs] = ch;
+        self.wofs = (self.wofs + 1) % (BUP_WINDOWSIZE as i32);
+    }
+
+    pub fn digest(&self) -> u32 {
+        (self.s1 << 16) | (self.s2 & 0xFFFF)
+    }
+}
+
+#[no_mangle]
+pub extern fn bupsplit_sum(buf: *const u8, ofs: libc::size_t, len: libc::size_t) -> u32 {
+    let sbuf = unsafe {
+        assert!(!buf.is_null());
+        slice::from_raw_parts(buf.offset(ofs as isize), (len - ofs) as usize)
+    };
+
+    let mut r = Rollsum::new();
+    for x in sbuf {
+        r.roll(*x);
+    }
+    r.digest()
+}
+
+#[no_mangle]
+pub extern fn bupsplit_find_ofs(buf: *const u8, len: libc::size_t,
+                                bits: *mut libc::c_int) -> libc::c_int
+{
+    let sbuf = unsafe {
+        assert!(!buf.is_null());
+        slice::from_raw_parts(buf, len as usize)
+    };
+
+    let mut r = Rollsum::new();
+    for x in sbuf {
+        r.roll(*x);
+	      if (r.s2 & (BUP_BLOBSIZE-1)) == ((u32::max_value()) & (BUP_BLOBSIZE-1)) {
+	          if !bits.is_null() {
+                let mut sum = r.digest() >> BUP_BLOBBITS;
+                let mut rbits : libc::c_int = BUP_BLOBBITS as i32;
+                while sum & 1 != 0 {
+                    sum = sum >> 1;
+                    rbits = rbits + 1;
+                }
+                unsafe {
+                    *bits = rbits;
+                }
+            }
+            return len as i32
+        }
+    }
+    0
+}