From c48e8bfad36134c8da054ae08566d0c3dd149412 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 Feb 2021 18:15:21 +0000 Subject: [PATCH] ci: Rework build/test dependency install Now that `ci/installdeps.sh` gracefully exits if run as non-root, we can fold the cargo bits into the our build scripts and avoid invoking both of them. However, now we need to split test deps to separate file because we won't have `cargo` in the main cosa pod. This also fixes a FIXME. Steal the `grep` invocation from cosa and make it a declarative text file so we can have comments per package etc. --- .cci.jenkinsfile | 5 ++--- ci/build.sh | 1 - ci/clang-analyzer.sh | 1 - ci/coreosci-rpmbuild.sh | 1 - ci/install-extra-builddeps.sh | 12 ------------ ci/install-test-deps.sh | 6 ++++++ ci/installdeps.sh | 20 ++++++++++++++------ ci/testdeps.txt | 8 ++++++++ ci/unit.sh | 1 - configure.ac | 2 +- packaging/make-git-snapshot.sh | 2 +- 11 files changed, 32 insertions(+), 27 deletions(-) create mode 100755 ci/install-test-deps.sh create mode 100644 ci/testdeps.txt diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 5cd9e699..d0ffa16f 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -50,8 +50,7 @@ parallel insttests: { """) } stage("Install Deps") { - // really, we just need test deps, but meh... - shwrap("ci/installdeps.sh") + shwrap("ci/install-test-deps.sh") } stage("Kola") { // TODO upstream this into coreos-ci-lib @@ -86,7 +85,7 @@ compose: { unstash 'rpms' stage("Install Deps") { shwrap(""" - ci/installdeps.sh # really, we just need test deps, but meh... + ci/install-test-deps.sh dnf install -y *.rpm # install our built rpm-ostree """) } diff --git a/ci/build.sh b/ci/build.sh index bdfd194f..719b9d80 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -6,7 +6,6 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libbuild.sh -${dn}/install-extra-builddeps.sh ${dn}/installdeps.sh # make it clear what rustc version we're compiling with (this is grepped in CI) rustc --version diff --git a/ci/clang-analyzer.sh b/ci/clang-analyzer.sh index c88d0be1..c9171cee 100755 --- a/ci/clang-analyzer.sh +++ b/ci/clang-analyzer.sh @@ -5,7 +5,6 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libbuild.sh -${dn}/install-extra-builddeps.sh ${dn}/installdeps.sh env NOCONFIGURE=1 ./autogen.sh scan-build ./configure --prefix=/usr --libdir=/usr/lib64 --sysconfdir=/etc diff --git a/ci/coreosci-rpmbuild.sh b/ci/coreosci-rpmbuild.sh index ed2d77c6..6f749ffa 100755 --- a/ci/coreosci-rpmbuild.sh +++ b/ci/coreosci-rpmbuild.sh @@ -8,7 +8,6 @@ dn=$(dirname $0) git fetch origin --tags git submodule update --init --recursive ci/installdeps.sh -ci/install-extra-builddeps.sh # Our primary CI build goes via RPM rather than direct to binaries # to better test that path, including our vendored spec file, etc. # The RPM build expects pre-generated bindings, so do that now. diff --git a/ci/install-extra-builddeps.sh b/ci/install-extra-builddeps.sh index 84f9fa9c..f1f641af 100755 --- a/ci/install-extra-builddeps.sh +++ b/ci/install-extra-builddeps.sh @@ -1,13 +1 @@ #!/usr/bin/env bash -# cxx.rs (cxxbridge) isn't packaged in Fedora today. Both it and cbindgen generate -# source code, which we vendor along with our dependent crates into release -# tarballs. Note in the future it's likely we stop using cbindgen entirely in -# favor of cxx.rs. - -set -euo pipefail - -CXX_VER=$(cargo metadata --format-version 1 | jq -r '.packages[]|select(.name == "cxx").version') -time cargo install cxxbridge-cmd --version "${CXX_VER}" - -CBINDGEN_VER=$(cargo metadata --format-version 1 | jq -r '.packages[]|select(.name == "cbindgen").version') -time cargo install cbindgen --version "${CBINDGEN_VER}" diff --git a/ci/install-test-deps.sh b/ci/install-test-deps.sh new file mode 100755 index 00000000..6e1b1c24 --- /dev/null +++ b/ci/install-test-deps.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail +dn=$(dirname $0) +. ${dn}/libbuild.sh +deps=$(grep -v '^#' "${dn}"/testdeps.txt) +pkg_install ${deps} diff --git a/ci/installdeps.sh b/ci/installdeps.sh index 71894d10..e494a500 100755 --- a/ci/installdeps.sh +++ b/ci/installdeps.sh @@ -6,6 +6,18 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libbuild.sh +# cxx.rs (cxxbridge) isn't packaged in Fedora today. Both it and cbindgen generate +# source code, which we vendor along with our dependent crates into release +# tarballs. Note in the future it's likely we stop using cbindgen entirely in +# favor of cxx.rs. +CXX_VER=$(cargo metadata --format-version 1 | jq -r '.packages[]|select(.name == "cxx").version') +time cargo install cxxbridge-cmd --version "${CXX_VER}" +CBINDGEN_VER=$(cargo metadata --format-version 1 | jq -r '.packages[]|select(.name == "cbindgen").version') +time cargo install cbindgen --version "${CBINDGEN_VER}" + +# Everything below here uses dnf/yum; we don't try to use +# sudo for you right now. (Though hopefully you're building +# in an unprivileged podman container at least) if [ -n "${SKIP_INSTALLDEPS:-}" ] || test $(id -u) != 0; then exit 0 fi @@ -13,9 +25,5 @@ fi # we have the canonical spec file handy so just builddep from that # XXX: use --allowerasing as a temporary hack to ease the migration to libmodulemd2 time dnf builddep --spec -y packaging/rpm-ostree.spec.in --allowerasing -# Mostly dependencies for tests; TODO move these into the spec file -# and also put them in the cosa buildroot (or another container) -pkg_install ostree{,-devel,-grub2} createrepo_c /usr/bin/jq python3-pyyaml \ - libubsan libasan libtsan elfutils fuse sudo python3-gobject-base \ - selinux-policy-devel selinux-policy-targeted python3-createrepo_c \ - rsync python3-rpm parallel clang rustfmt-preview distribution-gpg-keys + + diff --git a/ci/testdeps.txt b/ci/testdeps.txt new file mode 100644 index 00000000..0dce4002 --- /dev/null +++ b/ci/testdeps.txt @@ -0,0 +1,8 @@ +# Dependencies for our test suites, i.e. ./tests/compose and +# ./tests/vmcheck. +createrepo_c +/usr/bin/jq +python3-pyyaml +libubsan libasan libtsan elfutils fuse sudo python3-gobject-base +selinux-policy-devel selinux-policy-targeted python3-createrepo_c +rsync python3-rpm parallel distribution-gpg-keys diff --git a/ci/unit.sh b/ci/unit.sh index c958e8ca..a0821348 100755 --- a/ci/unit.sh +++ b/ci/unit.sh @@ -5,5 +5,4 @@ dn=$(dirname $0) . ${dn}/libbuild.sh ci/installdeps.sh -ci/install-extra-builddeps.sh ci/build.sh diff --git a/configure.ac b/configure.ac index 66f12c85..98a220ca 100644 --- a/configure.ac +++ b/configure.ac @@ -98,7 +98,7 @@ AS_IF([test -z "$cargo"], [AC_MSG_ERROR([cargo is required for --enable-rust])]) AC_PATH_PROG([rustc], [rustc]) AS_IF([test -z "$rustc"], [AC_MSG_ERROR([rustc is required for --enable-rust])]) -dnl See comment in install-extra-builddeps.sh +dnl See comment in installdeps.sh AM_CONDITIONAL(BUILDOPT_PREBUILT_BINDINGS, [test -f rpmostree-rust-prebuilt.h]) AC_MSG_CHECKING(whether to build in debug mode) diff --git a/packaging/make-git-snapshot.sh b/packaging/make-git-snapshot.sh index da374819..ccf8ecdf 100755 --- a/packaging/make-git-snapshot.sh +++ b/packaging/make-git-snapshot.sh @@ -60,7 +60,7 @@ open(checksum_file, "w").write(json.dumps(j))' $crate_subdir tar --transform="s,^,${PKG_VER}/," -rf ${TARFILE_TMP} * .cargo/ ) -# And finally, vendor generated code. See install-extra-builddeps.sh +# And finally, vendor generated code. See installdeps.sh # and Makefile-rpm-ostree.am for more. (cd ${srcdir} cp rpmostree-rust{,-prebuilt}.h