diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index a76b331d..36daf6f3 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -79,7 +79,8 @@ stage("Build FCOS") { } -stage("Run vmcheck") { +stage("Test") { +parallel vmcheck: { def nhosts = 6 def mem = (nhosts * 1024) + 512 coreos.pod(image: COSA_IMAGE, runAsUser: 0, kvm: true, memory: "${mem}Mi", cpu: "${nhosts}") { @@ -100,7 +101,7 @@ stage("Run vmcheck") { set -xeuo pipefail fcos=\$(ls builds/latest/*/*.qcow2) # */ ln -sf "\$(realpath \${fcos})" tests/vmcheck/image.qcow2 - NHOSTS=${nhosts} tests/vmcheck.sh + JOBS=${nhosts} tests/vmcheck.sh """ } } finally { @@ -112,4 +113,36 @@ stage("Run vmcheck") { archiveArtifacts allowEmptyArchive: true, artifacts: 'vmcheck-logs.tar.xz' } } -} +}, +compose: { + def jobs = 5 + def mem = (jobs * 2048) + 512 + coreos.pod(image: COSA_IMAGE, runAsUser: 0, emptyDirs: ["/srv/tmpdir"], kvm: true, memory: "${mem}Mi", cpu: "${jobs}") { + checkout scm + unstash 'rpms' + sh """ + set -euo pipefail + ci/installdeps.sh # really, we just need test deps, but meh... + + # install our built rpm-ostree + find packaging/ ! -name '*.src.rpm' -name '*.rpm' | xargs dnf install -y + rm -rf packaging + """ + try { + timeout(time: 40, unit: 'MINUTES') { + sh """ + set -xeuo pipefail + mkdir compose-logs + TMPDIR=/srv/tmpdir JOBS=${jobs} ./tests/compose.sh + """ + } + } finally { + sh """ + if [ -d compose-logs ]; then + tar -C compose-logs -cf- . | xz -c9 > compose-logs.tar.xz + fi + """ + archiveArtifacts allowEmptyArchive: true, artifacts: 'compose-logs.tar.xz' + } + } +}} diff --git a/.papr.yml b/.papr.yml deleted file mode 100644 index 2d47fcf6..00000000 --- a/.papr.yml +++ /dev/null @@ -1,46 +0,0 @@ -branches: - - master - - auto - - try - -# NB: when bumping 29 here, also bump compose script - -context: f29-compose1 - -build: false - -timeout: 35m - -# This test case wants an "unprivileged container with bubblewrap", -# which we don't have right now; so just provision a VM and do a -# docker --privileged run. -host: - distro: fedora/29/atomic - # Compose tests are slow and should be parallelized - specs: - cpus: 4 - -env: - RPMOSTREE_COMPOSE_TEST_FILTER: odd - -# Copy yum.repos.d to get any injected repos from the host, which -# will point to a closer mirror. Note we substitute $releasever -# since https://github.com/projectatomic/rpm-ostree/pull/875 -tests: - - docker run --privileged --rm - -e RPMOSTREE_COMPOSE_TEST_FILTER - -e RPMOSTREE_COMPOSE_TEST_USE_REPOS=/etc/yum.repos.d.host - -v /etc/yum.repos.d:/etc/yum.repos.d.host:ro - -v $(pwd):/srv/code -w /srv/code - registry.fedoraproject.org/fedora:29 /bin/sh -c - "cp /etc/yum.repos.d.host/* /etc/yum.repos.d/ && ./ci/build.sh && make install && ./tests/compose" - -artifacts: - - test-compose-logs - ---- - -inherit: true -context: f29-compose2 -env: - RPMOSTREE_COMPOSE_TEST_FILTER: even diff --git a/tests/common/libtest-core.sh b/tests/common/libtest-core.sh index fc5439d7..e71e3881 100644 --- a/tests/common/libtest-core.sh +++ b/tests/common/libtest-core.sh @@ -133,3 +133,51 @@ skip() { echo "1..0 # SKIP" "$@" exit 0 } + +# https://github.com/coreos/coreos-assembler/pull/632. Ideally, we'd also cap +# based on memory available to us, but that's notoriously difficult to do for +# containers (see: +# https://fabiokung.com/2014/03/13/memory-inside-linux-containers/). We make an +# assumption here that we have at least e.g. 1G of RAM we can use per CPU +# available to us. +ncpus() { + if ! grep -q kubepods /proc/1/cgroup; then + # this might be a developer laptop; leave one cpu free to be nice + echo $(($(nproc) - 1)) + return 0 + fi + + quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us) + period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us) + if [[ ${quota} != -1 ]] && [[ ${period} -gt 0 ]]; then + echo $(("${quota}" / "${period}")) + fi + + # just fallback to 1 + echo 1 +} + +filter_tests() { + local tests_dir=$1; shift + local skipped=0 + + local selected_tests=() + for tf in $(find "${tests_dir}" -name 'test-*.sh' | shuf); do + tfbn=$(basename "$tf" .sh) + tfbn=" ${tfbn#test-} " + if [ -n "${TESTS+ }" ]; then + if [[ " $TESTS " != *$tfbn* ]]; then + skipped=$((skipped + 1)) + continue + fi + fi + + selected_tests+=("${tfbn}") + done + + if [ ${skipped} -gt 0 ]; then + echo "Skipping ${skipped} tests" >&2 + fi + + echo "${selected_tests[*]}" +} diff --git a/tests/common/libtest.sh b/tests/common/libtest.sh index 6aceea7a..f7f6d568 100644 --- a/tests/common/libtest.sh +++ b/tests/common/libtest.sh @@ -51,12 +51,14 @@ _cleanup_tmpdir () { # Create a tmpdir if we're running as a local test (i.e. through `make check`) # or as a `vmcheck` test, which also needs some scratch space on the host. -if ( test -n "${UNINSTALLEDTESTS:-}" || test -n "${VMTESTS:-}" ) && ! test -f $PWD/.test; then +if { test -n "${UNINSTALLEDTESTS:-}" || \ + test -n "${VMTESTS:-}" || \ + test -n "${COMPOSETESTS:-}"; } && ! test -f "$PWD/.test"; then # Use --tmpdir to keep it in /tmp. This also keeps paths short; this is # important if we want to create UNIX sockets under there. test_tmpdir=$(mktemp -d test.XXXXXX --tmpdir) touch ${test_tmpdir}/.test - trap _cleanup_tmpdir EXIT + trap _cleanup_tmpdir EXIT SIGINT cd ${test_tmpdir} fi if test -n "${UNINSTALLEDTESTS:-}"; then @@ -404,6 +406,8 @@ EOF case $section in requires) echo "Requires: $arg" >> $spec;; + recommends) + echo "Recommends: $arg" >> $spec;; provides) echo "Provides: $arg" >> $spec;; conflicts) @@ -577,3 +581,26 @@ assert_jq() { fi done } + +# This function below was taken and adapted from coreos-assembler. We +# should look into sharing this code more easily. + +# Determine if current user has enough privileges for composes +_privileged= +has_compose_privileges() { + if [ -z "${_privileged:-}" ]; then + if [ -n "${FORCE_UNPRIVILEGED:-}" ]; then + echo "Detected FORCE_UNPRIVILEGED; using virt" + _privileged=0 + elif ! capsh --print | grep -q 'Bounding.*cap_sys_admin'; then + echo "Missing CAP_SYS_ADMIN; using virt" + _privileged=0 + elif [ "$(id -u)" != "0" ]; then + echo "Not running as root; using virt" + _privileged=0 + else + _privileged=1 + fi + fi + [ ${_privileged} == 1 ] +} diff --git a/tests/compose.sh b/tests/compose.sh index cb6fca5e..9ad6bffc 100755 --- a/tests/compose.sh +++ b/tests/compose.sh @@ -1,96 +1,144 @@ #!/bin/bash set -euo pipefail -dn=$(cd $(dirname $0) && pwd) +# freeze on a specific commit for tests for reproducibility and since it should +# always work to target older treefiles +FEDORA_COREOS_CONFIG_COMMIT=088fc2dec535aca392958e9c30c17cf19ef4b568 -export topsrcdir=$(cd $dn/.. && pwd) -. ${dn}/common/libtest-core.sh -. ${dn}/common/libtestrepos.sh +dn=$(cd "$(dirname "$0")" && pwd) +topsrcdir=$(cd "$dn/.." && pwd) +commondir=$(cd "$dn/common" && pwd) +export topsrcdir commondir -# avoid refetching yum metadata everytime -export RPMOSTREE_USE_CACHED_METADATA=1 +# shellcheck source=common/libtest-core.sh +. "${commondir}/libtest.sh" -export LOGDIR=${LOGDIR:-$(pwd)/test-compose-logs} -mkdir -p ${LOGDIR} - -colour_print() { - colour=$1; shift - [ ! -t 1 ] || echo -en "\e[${colour}m" - echo -n "$@" - [ ! -t 1 ] || echo -en "\e[0m" - echo -} - -pass_print() { - colour_print 32 "$@" # green -} - -fail_print() { - colour_print 31 "$@" # red -} - -skip_print() { - colour_print 34 "$@" # blue -} - -uid=$(id -u) -test_compose_datadir=/var/tmp/rpmostree-compose-cache-${uid} -export test_compose_datadir -mkdir -p ${test_compose_datadir} -datadir_owner=$(stat -c '%u' ${test_compose_datadir}) -test ${uid} = ${datadir_owner} - -# Create a consistent cache of the RPMs -echo "Preparing compose tests... $(date)" -tmp_repo=${test_compose_datadir}/tmp-repo -if test -z "${RPMOSTREE_COMPOSE_CACHEONLY:-}"; then - setup_rpmmd_repos ${dn}/composedata - ostree --repo=${tmp_repo} init --mode=bare-user - # Ensure all subsequent tests have the RPMs - mkdir -p ${test_compose_datadir}/{fedora-local,cache} - rpm-ostree compose --repo=${tmp_repo} tree --download-only-rpms --cachedir=${test_compose_datadir}/cache ${dn}/composedata/fedora-base.json - find ${test_compose_datadir}/cache/ -name '*.rpm' | while read f; do - mv $f ${test_compose_datadir}/fedora-local - done - (cd ${test_compose_datadir}/fedora-local && createrepo_c .) -fi -echo "Done preparing compose tests! $(date)" -rm ${tmp_repo} -rf - -total=0 -pass=0 -fail=0 -skip=0 -all_tests="$(cd ${dn}/compose-tests && ls test-*.sh | sort)" -if [ "${RPMOSTREE_COMPOSE_TEST_FILTER:-}" == odd ]; then - # https://superuser.com/a/101760/237392 - all_tests="$(sed -n 'p;n' <<< ${all_tests})" -elif [ "${RPMOSTREE_COMPOSE_TEST_FILTER:-}" == even ]; then - all_tests="$(sed -n 'n;p' <<< ${all_tests})" -fi -tests="" -if [ -n "${TESTS+ }" ]; then - for tf in ${all_tests}; do - tfbn=$(basename "$tf" .sh) - tfbn=" ${tfbn#test-} " - if [[ " $TESTS " != *$tfbn* ]]; then - echo "Skipping: ${tf}" - continue - fi - tests="${tests} ${tf}" - done -else - tests="${all_tests}" +read -r -a tests <<< "$(filter_tests "${topsrcdir}/tests/compose")" +if [ ${#tests[*]} -eq 0 ]; then + echo "No tests selected; mistyped filter?" + exit 0 fi -if test -z "${tests}"; then - fatal "error: No tests match ${TESTS}" +JOBS=${JOBS:-$(ncpus)} + +outputdir="${topsrcdir}/compose-logs" +fixtures="$(pwd)/compose-cache" + +# re-use the same FCOS config and RPMs if it already exists +if [ ! -d compose-cache ]; then + mkdir -p compose-cache + + # first, download all the RPMs into a directory + echo "Caching test fixtures in compose-cache/" + + # Really want to use cosa fetch for this and just share the pkgcache repo. + # Though for now we still need to support non-unified mode. Once we don't, we + # can clean this up. + pushd compose-cache + git clone https://github.com/coreos/fedora-coreos-config config + + pushd config + git checkout "${FEDORA_COREOS_CONFIG_COMMIT}" + # we flatten the treefile to make it easier to manipulate in tests (we have + # lots of tests that check for include logic already) + rpm-ostree compose tree --print-only manifest.yaml > manifest.json + rm manifest.yaml + mv manifests/{passwd,group} . + rm -rf manifests/ + popd + + if ! has_compose_privileges; then + # Unlike cosa, we don't need as much flexibility since we don't e.g. build + # images. So just create the supermin appliance and root now so each test + # doesn't have to build it. + mkdir -p supermin.{prepare,build} + # we just import the strict minimum here that rpm-ostree needs + rpms="rpm-ostree bash rpm-build coreutils selinux-policy-targeted dhcp-client util-linux" + # shellcheck disable=SC2086 + supermin --prepare --use-installed -o supermin.prepare $rpms + # the reason we do a heredoc here is so that the var substition takes + # place immediately instead of having to proxy them through to the VM + cat > init < tmp/cmd.sh.rc +if [ -b /dev/sdb1 ]; then + /sbin/fstrim -v cache +fi +/sbin/reboot -f +EOF + chmod a+x init + tar -czf supermin.prepare/init.tar.gz --remove-files init + supermin --build "${fixtures}/supermin.prepare" --size 5G -f ext2 -o supermin.build + fi + + mkdir cachedir + # we just need a repo so we can download stuff (but see note above about + # sharing pkgcache repo in the future) + ostree init --repo=repo --mode=archive + rpm-ostree compose tree --unified-core --download-only-rpms --repo=repo \ + config/manifest.json --cachedir cachedir \ + --ex-lockfile config/manifest-lock.x86_64.json \ + --ex-lockfile config/manifest-lock.overrides.x86_64.yaml + rm -rf repo + (cd cachedir && createrepo_c .) + echo -e "[cache]\nbaseurl=$(pwd)/cachedir\ngpgcheck=0" > config/cache.repo + + pushd config + python3 -c ' +import sys, json +y = json.load(sys.stdin) +y["repos"] = ["cache"] +json.dump(y, sys.stdout)' < manifest.json > manifest.json.new + mv manifest.json{.new,} + git add . + git -c user.email="composetest@localhost.com" -c user.name="composetest" \ + commit -am 'modifications for tests' + popd + + popd fi -echo "Compose tests starting: $(date)" -echo "Executing: ${tests}" -echo "Writing logs to ${LOGDIR}" -(for tf in ${tests}; do echo $tf; done) | \ - parallel -v -j +1 --progress --halt soon,fail=1 \ - --results ${LOGDIR}/parallel --quote /bin/sh -c "${dn}/compose-tests/run-test.sh {}" -echo "$(date): All tests passed" +echo "Running ${#tests[*]} tests ${JOBS} at a time" + +echo "Test results outputting to ${outputdir}/" + +echo -n "${tests[*]}" | parallel -d' ' -j "${JOBS}" --line-buffer \ + "${topsrcdir}/tests/compose/runtest.sh" "${outputdir}" "${fixtures}" diff --git a/tests/compose/libbasic-test.sh b/tests/compose/libbasic-test.sh index a7786c40..0ebd49df 100644 --- a/tests/compose/libbasic-test.sh +++ b/tests/compose/libbasic-test.sh @@ -1,6 +1,6 @@ # This used to live in test-basic.sh, but it's now shared with test-basic-unified.sh basic_test() { -if ostree --repo=${repobuild} ls -R ${treeref} /usr/etc/passwd-; then +if ostree --repo=${repo} ls -R ${treeref} /usr/etc/passwd-; then assert_not_reached "Found /usr/etc/passwd- backup file in tree" fi echo "ok passwd no backups" @@ -8,8 +8,8 @@ echo "ok passwd no backups" validate_passwd() { f=$1 shift - ostree --repo=${repobuild} cat ${treeref} /usr/lib/$f |grep -v '^root' | sort > $f.tree - cat composedata/$f | while read line; do + ostree --repo=${repo} cat ${treeref} /usr/lib/$f |grep -v '^root' | sort > $f.tree + cat config/$f | while read line; do if ! grep -q "$line" "$f.tree"; then echo "Missing entry: %line" fi @@ -19,97 +19,93 @@ validate_passwd() { validate_passwd passwd validate_passwd group - -ostree --repo=${repobuild} cat ${treeref} /usr/etc/default/useradd > useradd.txt +ostree --repo=${repo} cat ${treeref} /usr/etc/default/useradd > useradd.txt assert_file_has_content_literal useradd.txt HOME=/var/home -ostree --repo=${repobuild} cat ${treeref} \ +ostree --repo=${repo} cat ${treeref} \ /usr/etc/selinux/targeted/contexts/files/file_contexts.homedirs > homedirs.txt assert_file_has_content homedirs.txt '^/var/home' -ostree --repo=${repobuild} cat ${treeref} \ +ostree --repo=${repo} cat ${treeref} \ /usr/etc/selinux/targeted/contexts/files/file_contexts.subs_dist > subs_dist.txt assert_not_file_has_content subs_dist.txt '^/var/home \+' assert_file_has_content subs_dist.txt '^/home \+/var/home$' echo "ok etc/default/useradd" for path in /usr/share/rpm /usr/lib/sysimage/rpm-ostree-base-db; do - ostree --repo=${repobuild} ls -R ${treeref} ${path} > db.txt + ostree --repo=${repo} ls -R ${treeref} ${path} > db.txt assert_file_has_content_literal db.txt /Packages done echo "ok db" -ostree --repo=${repobuild} show --print-metadata-key exampleos.gitrepo ${treeref} > meta.txt +ostree --repo=${repo} show --print-metadata-key exampleos.gitrepo ${treeref} > meta.txt assert_file_has_content meta.txt 'rev.*97ec21c614689e533d294cdae464df607b526ab9' assert_file_has_content meta.txt 'src.*https://gitlab.com/exampleos/custom-atomic-host' -ostree --repo=${repobuild} show --print-metadata-key exampleos.tests ${treeref} > meta.txt +ostree --repo=${repo} show --print-metadata-key exampleos.tests ${treeref} > meta.txt assert_file_has_content meta.txt 'smoketested.*e2e' -ostree --repo=${repobuild} show --print-metadata-key rpmostree.rpmmd-repos ${treeref} > meta.txt -assert_file_has_content meta.txt 'id.*fedora.*timestamp' -ostree --repo=${repobuild} show --print-metadata-key foobar ${treeref} > meta.txt +ostree --repo=${repo} show --print-metadata-key rpmostree.rpmmd-repos ${treeref} > meta.txt +assert_file_has_content meta.txt 'id.*cache.*timestamp' +ostree --repo=${repo} show --print-metadata-key foobar ${treeref} > meta.txt assert_file_has_content meta.txt 'bazboo' -ostree --repo=${repobuild} show --print-metadata-key overrideme ${treeref} > meta.txt +ostree --repo=${repo} show --print-metadata-key overrideme ${treeref} > meta.txt assert_file_has_content meta.txt 'new val' echo "ok metadata" -ostree --repo=${repobuild} ls -R ${treeref} /usr/lib/ostree-boot > bootls.txt -assert_file_has_content bootls.txt vmlinuz- -assert_file_has_content bootls.txt initramfs- -echo "ok boot files" -vmlinuz_line=$(grep -o '/vmlinuz.*$' bootls.txt) -kver=$(echo ${vmlinuz_line} | sed -e 's,^/vmlinuz-,,' -e 's,-[0-9a-f]*$,,') -ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/vmlinuz >/dev/null -ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/initramfs.img >ls.txt -assert_file_has_content ls.txt '^-00644' +ostree --repo=${repo} ls -R ${treeref} /usr/lib/modules | grep -v /kernel/ > ls.txt +assert_file_has_content ls.txt '/vmlinuz$' +assert_file_has_content ls.txt '^-00644 .*/initramfs.img$' +echo "ok kernel and initramfs" -ostree --repo=${repobuild} ls -R ${treeref} /usr/share/man > manpages.txt -assert_file_has_content manpages.txt man5/ostree.repo.5 -echo "ok manpages" +ostree --repo=${repo} ls ${treeref} /usr/share > share.txt +assert_not_file_has_content share.txt /usr/share/man +# test-misc-tweaks tests the docs path +echo "ok no manpages" # https://github.com/projectatomic/rpm-ostree/pull/1425 -ostree --repo=${repobuild} ls ${treeref} /usr/etc/machine-id -echo "ok machine-id" +ostree --repo=${repo} ls ${treeref} /usr/etc > ls.txt +assert_not_file_has_content ls.txt 'machine-id' +# test-misc-tweaks tests the machine-id compat path +echo "ok no machine-id" -ostree --repo=${repobuild} ls ${treeref} usr/etc/systemd/system/multi-user.target.wants/chronyd.service > preset.txt +ostree --repo=${repo} ls ${treeref} usr/etc/systemd/system/multi-user.target.wants/chronyd.service > preset.txt assert_file_has_content_literal preset.txt '-> /usr/lib/systemd/system/chronyd.service' echo "ok systemctl preset" -ostree --repo=${repobuild} ls -X ${treeref} usr/bin/docker-current > docker.txt +ostree --repo=${repo} ls -X ${treeref} usr/bin/docker > docker.txt assert_file_has_content_literal docker.txt 'system_u:object_r:container_runtime_exec_t:s0' echo "ok container-selinux" -ostree --repo=${repobuild} ls ${treeref} /usr/bin/su > su.txt +ostree --repo=${repo} ls ${treeref} /usr/bin/su > su.txt assert_file_has_content su.txt '^-04[71][0-7][0-7]' echo "ok setuid" -ostree --repo=${repobuild} ls -X ${treeref} /usr/bin/ping > ping.txt +ostree --repo=${repo} ls -X ${treeref} /usr/bin/ping > ping.txt assert_file_has_content_literal ping.txt "b'security.capability', [byte" echo "ok fcaps" # https://github.com/projectatomic/rpm-ostree/issues/669 -ostree --repo=${repobuild} ls ${treeref} /tmp > ls.txt -assert_file_has_content ls.txt 'l00777 0 0 0 /tmp -> sysroot/tmp' +ostree --repo=${repo} ls ${treeref} /tmp > ls.txt +assert_file_has_content ls.txt 'd01777 0 0 0 /tmp' echo "ok /tmp" -ostree --repo=${repobuild} ls ${treeref} /usr/share/rpm > ls.txt +ostree --repo=${repo} ls ${treeref} /usr/share/rpm > ls.txt assert_not_file_has_content ls.txt '__db' 'lock' -ostree --repo=${repobuild} ls -R ${treeref} /usr/etc/selinux > ls.txt +ostree --repo=${repo} ls -R ${treeref} /usr/etc/selinux > ls.txt assert_not_file_has_content ls.txt 'LOCK' echo "ok no leftover files" -ostree --repo=${repobuild} show ${treeref} \ +ostree --repo=${repo} show ${treeref} \ --print-metadata-key rpmostree.rpmdb.pkglist > pkglist.txt assert_file_has_content pkglist.txt 'systemd' -# This is currently a Recommends: package. If you change this, please -# also change the corresponding test in misc-tweaks.sh. -assert_file_has_content pkglist.txt 'systemd-bootchart' +assert_file_has_content_literal pkglist.txt 'foobar' +assert_not_file_has_content pkglist.txt 'foobar-rec' echo "ok compose pkglist" -ostree --repo=${repobuild} cat ${treeref} /usr/share/rpm-ostree/treefile.json > treefile.json +ostree --repo=${repo} cat ${treeref} /usr/share/rpm-ostree/treefile.json > treefile.json assert_jq treefile.json '.basearch == "x86_64"' echo "ok basearch" -ostree --repo=${repobuild} rev-parse ${treeref}^ > parent.txt +ostree --repo=${repo} rev-parse ${treeref}^ > parent.txt assert_file_has_content parent.txt 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b echo "ok --parent" } diff --git a/tests/compose/libcomposetest.sh b/tests/compose/libcomposetest.sh index 7edbd640..39e7cb9a 100644 --- a/tests/compose/libcomposetest.sh +++ b/tests/compose/libcomposetest.sh @@ -1,82 +1,127 @@ -dn=$(cd $(dirname $0) && pwd) -test_tmpdir=$(mktemp -d ${RPMOSTREE_TMPDIR_LOCATION:-/var/tmp}/rpm-ostree-compose-test.XXXXXX) -touch ${test_tmpdir}/.test -trap _cleanup_tmpdir EXIT -cd ${test_tmpdir} -. ${dn}/../common/libtest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=../common/libtest.sh +. "${dn}/../common/libtest.sh" -export repo=$(pwd)/repo -export repobuild=$(pwd)/repo-build +export repo=$PWD/repo +export treefile=$PWD/config/manifest.json +treeref=$(jq -r .ref < "${treefile}"); export treeref -pyeditjson() { - cat >editjson.py <pyedit.py < "${f}.new" + rm -f ./pyedit.py + mv "${f}"{.new,} } -pysetjsonmember() { - pyeditjson "jd['"$1"'] = $2" < ${treefile} > ${treefile}.new && mv ${treefile}{.new,} +treefile_pyedit() { + pyedit "${treefile}" "$@" } -pyappendjsonmember() { - pyeditjson "jd['"$1"'] += $2" < ${treefile} > ${treefile}.new && mv ${treefile}{.new,} +treefile_set() { + treefile_pyedit "tf['""$1""'] = $2" } -prepare_compose_test() { - name=$1 - shift - filetype=${1:-json} - ostree --repo=${repo} init --mode=archive - echo 'fsync=false' >> ${repo}/config - ostree --repo=${repobuild} init --mode=bare-user - echo 'fsync=false' >> ${repobuild}/config - mkdir -p ${test_compose_datadir}/cache - cp -r ${dn}/../composedata . - # We use the local RPM package cache - rm -f composedata/*.repo - cat > composedata/fedora-local.repo < ${treefile} - pysetjsonmember "repos" '["fedora-local"]' ${treefile} - # FIXME extract from json - if [ "${filetype}" = "yaml" ]; then - python3 < tmp.json + treeref=$(jq -r .ref < tmp.json); export treeref + rm tmp.json +} + +treefile_append() { + treefile_pyedit " +if '$1' not in tf: + tf['$1'] = $2 +else: + tf['$1'] += $2" +} + +# for tests that need direct control on rpm-ostree +export compose_base_argv="\ + --unified-core \ + --repo=${repo} \ + --cachedir=${test_tmpdir}/cache \ + --ex-lockfile=config/manifest-lock.x86_64.json \ + --ex-lockfile=config/manifest-lock.overrides.x86_64.yaml" + +# and create this now for tests which only use `compose_base_argv` +mkdir -p cache + +runcompose() { + # keep this function trivial and the final command runasroot to mostly steer + # clear of huge footgun of set -e not working in function calls in if-stmts + runasroot rpm-ostree compose tree ${compose_base_argv} \ + --write-composejson-to=compose.json "${treefile}" "$@" +} + +# NB: One difference from cosa here is we don't use `sudo`. I think there's an +# issue with sudo under parallel not getting signals propagated from the +# controlling terminal? Anyway, net result is we can end up with a bunch of +# rpm-ostree processes leaking in the background still running. So for now, one +# has to run this testsuite as root, or use unprivileged. XXX: to investigate. + +runasroot() { + if has_compose_privileges; then + "$@" + else + runvm "$@" fi } -composejson=$(pwd)/compose.json -compose_workdir=${test_tmpdir}/workdir -compose_base_argv="--workdir ${compose_workdir} --repo ${repobuild} --write-composejson-to ${composejson}" -runcompose() { - echo "$(date): starting compose" - # The workdir will be cleaned up (or not) with the overall test dir - rm ${compose_workdir} -rf - mkdir ${test_tmpdir}/workdir - env RPMOSTREE_PRESERVE_TMPDIR=1 rpm-ostree compose tree ${compose_base_argv} ${treefile} "$@" - commit=$(jq -r '.["ostree-commit"]' < "${composejson}") - ostree --repo=${repo} pull-local ${repobuild} "${treeref:-${commit}}" - echo "$(date): finished compose" -} +# This function below was taken and adapted from coreos-assembler. We +# should look into sharing this code more easily. -prepare_run_compose() { - prepare_compose_test $1 - runcompose +runvm() { + if [ ! -f tmp/cache.qcow2 ]; then + mkdir -p tmp + qemu-img create -f qcow2 tmp/cache.qcow2 8G + LIBGUESTFS_BACKEND=direct virt-format --filesystem=xfs -a tmp/cache.qcow2 + fi + + echo "export test_tmpdir=${test_tmpdir}" > tmp/env + # automatically proxy RPMOSTREE env vars + $(env | (grep ^RPMOSTREE || :) | xargs -r echo export) >> tmp/env + echo "$@" > tmp/cmd.sh + + #shellcheck disable=SC2086 + qemu-kvm \ + -nodefaults -nographic -m 2048 -no-reboot -cpu host \ + -kernel "${fixtures}/supermin.build/kernel" \ + -initrd "${fixtures}/supermin.build/initrd" \ + -netdev user,id=eth0,hostname=supermin \ + -device virtio-net-pci,netdev=eth0 \ + -device virtio-scsi-pci,id=scsi0,bus=pci.0,addr=0x3 \ + -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \ + -drive if=none,id=drive-scsi0-0-0-0,snapshot=on,file="${fixtures}/supermin.build/root" \ + -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=1 \ + -drive if=none,id=drive-scsi0-0-0-1,discard=unmap,file=tmp/cache.qcow2 \ + -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1 \ + -virtfs local,id=cache,path="${fixtures}",security_model=none,mount_tag=cache \ + -virtfs local,id=testdir,path="${test_tmpdir}",security_model=none,mount_tag=testdir \ + -serial stdio -append "root=/dev/sda console=ttyS0 selinux=1 enforcing=0 autorelabel=1" + + if [ ! -f tmp/cmd.sh.rc ]; then + fatal "Couldn't find rc file, something went terribly wrong!" + fi + return "$(cat tmp/cmd.sh.rc)" } diff --git a/tests/compose/run-test.sh b/tests/compose/run-test.sh deleted file mode 100755 index 010d5b3b..00000000 --- a/tests/compose/run-test.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -euo pipefail - -tf=$1 -export TEST_ARTIFACTS=${LOGDIR}/${tf} -mkdir -p ${TEST_ARTIFACTS} -# Redirect our stdout/stderr, since we don't want what GNU parallel does -exec 1>${TEST_ARTIFACTS}/output.txt -exec 2>&1 -# Rename the dir itself if non-zero rc to make it easy to know what failed -rc=0 -$(dirname $0)/${tf} || rc=$? -if [ $rc == 0 ]; then - mv ${TEST_ARTIFACTS}{,.pass} -else - mv ${TEST_ARTIFACTS}{,.fail.$rc} -fi -[ $rc == 0 ] diff --git a/tests/compose/runtest.sh b/tests/compose/runtest.sh new file mode 100755 index 00000000..5782340e --- /dev/null +++ b/tests/compose/runtest.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -euo pipefail + +if [ -n "${V:-}" ]; then + set -x +fi + +outputdir=$1; shift +fixtures=$1; shift +testname=$1; shift + +# this is used directly just by the basic test, but it also hosts the RPMs +export fixtures + +outputdir="${outputdir}/${testname}" +rm -rf "${outputdir:?}"/* +mkdir -p "${outputdir}" + +# keep original stdout around; this propagates to the terminal +exec 3>&1 + +# but redirect everything else to a log file +exec 1>"${outputdir}/output.log" +exec 2>&1 + +# seed output log with current date +date + +if [ -n "${V:-}" ]; then + setpriv --pdeathsig SIGKILL -- tail -f "${outputdir}/output.log" >&3 & +fi + +echo "EXEC: ${testname}" >&3 + +# this will cause libtest.sh to allocate a tmpdir and cd to it +export COMPOSETESTS=1 + +# shellcheck source=../common/libtest.sh disable=2154 +. "${commondir}/libtest.sh" + +# use `git clone` rather than a symlink; we want our own copy so that we can +# modify it +git clone file://${fixtures}/config +ostree init --repo repo --mode=bare-user + +if "${topsrcdir}/tests/compose/test-${testname}.sh"; then + echo "PASS: ${testname}" >&3 +else + echo "FAIL: ${testname}" >&3 + if [ -z "${V:-}" ]; then + tail -n20 "${outputdir}/output.log" | sed "s/^/ ${testname}: /g" >&3 + fi + + if [ -n "${COMPOSE_DEBUG:-}" ]; then + echo "--- COMPOSE_DEBUG ---" >&3 + echo "Working directory: ${PWD}" >&3 + echo "Sleeping..." >&3 + sleep infinity + fi + exit 1 +fi diff --git a/tests/compose/test-basic-unified.sh b/tests/compose/test-basic-unified.sh index 40fb1f89..2b44fbf2 100755 --- a/tests/compose/test-basic-unified.sh +++ b/tests/compose/test-basic-unified.sh @@ -1,19 +1,34 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "basic-unified" -# Test --print-only, currently requires --repo. We also +# Add a local rpm-md repo so we can mutate local test packages +treefile_append "repos" '["test-repo"]' +# test `recommends: false` (test-misc-tweaks tests the true path) +build_rpm foobar recommends foobar-rec +build_rpm foobar-rec + +echo gpgcheck=0 >> yumrepo.repo +ln "$PWD/yumrepo.repo" config/yumrepo.repo +treefile_append "packages" '["foobar"]' + +# Test --print-only. We also # just in this test (for now) use ${basearch} to test substitution. -pysetjsonmember "ref" '"fedora/stable/${basearch}/basic-unified"' -rpm-ostree compose tree --repo=${repobuild} --print-only ${treefile} > treefile.json +# shellcheck disable=SC2016 +treefile_set_ref '"fedora/stable/${basearch}/basic-unified"' +rpm-ostree compose tree --print-only "${treefile}" > treefile.json + # Verify it's valid JSON jq -r .ref < treefile.json > ref.txt # Test substitution of ${basearch} assert_file_has_content_literal ref.txt "${treeref}" + +treefile_pyedit "tf['add-commit-metadata']['foobar'] = 'bazboo'" +treefile_pyedit "tf['add-commit-metadata']['overrideme'] = 'old var'" + # Test metadata json with objects, arrays, numbers cat > metadata.json < metadata.json < autovar.txt +ostree --repo="${repo}" cat "${treeref}" /usr/lib/tmpfiles.d/pkg-filesystem.conf > autovar.txt # Picked this one at random as an example of something that won't likely be # converted to tmpfiles.d upstream. But if it is, we can change this test. assert_file_has_content_literal autovar.txt 'd /var/cache 0755 root root - -' -ostree --repo=${repobuild} cat ${treeref} /usr/lib/tmpfiles.d/pkg-chrony.conf > autovar.txt +ostree --repo="${repo}" cat "${treeref}" /usr/lib/tmpfiles.d/pkg-chrony.conf > autovar.txt # And this one has a non-root uid assert_file_has_content_literal autovar.txt 'd /var/log/chrony 0755 chrony chrony - -' # see rpmostree-importer.c -if ostree --repo=${repobuild} cat ${treeref} /usr/lib/tmpfiles.d/pkg-rpm.conf > rpm.txt 2>/dev/null; then +if ostree --repo="${repo}" cat "${treeref}" /usr/lib/tmpfiles.d/pkg-rpm.conf > rpm.txt 2>/dev/null; then assert_not_file_has_content rpm.txt 'd /var/lib/rpm' fi echo "ok autovar" -# And verify we're not hardlinking zero-sized files since this path isn't using -# rofiles-fuse -co=${repobuild}/tmp/usr-etc -ostree --repo=${repobuild} checkout -UHz --subpath=/usr/etc ${treeref} ${co} -# Verify the files exist and are zero-sized -for f in ${co}/sub{u,g}id; do - test -f "$f" - test '!' -s "$f" -done -if files_are_hardlinked ${co}/sub{u,g}id; then - fatal "Hardlinked zero-sized files without cachedir" -fi -rm ${co} -rf -echo "ok no cachedir zero-sized hardlinks" - # And redo it to trigger relabeling. Also test --no-parent at the same time. -origrev=$(ostree --repo=${repobuild} rev-parse ${treeref}) -runcompose --force-nocache --ex-unified-core --no-parent -newrev=$(ostree --repo=${repobuild} rev-parse ${treeref}) +origrev=$(ostree --repo="${repo}" rev-parse "${treeref}") +runcompose --force-nocache --no-parent +newrev=$(ostree --repo="${repo}" rev-parse "${treeref}") assert_not_streq "${origrev}" "${newrev}" echo "ok rerun" # And check that --no-parent worked. -if ostree rev-parse --repo "${repobuild}" ${newrev}^ 2>error.txt; then +if ostree rev-parse --repo "${repo}" "${newrev}"^ 2>error.txt; then assert_not_reached "New revision has a parent even with --no-parent?" fi assert_file_has_content_literal error.txt 'has no parent' diff --git a/tests/compose/test-basic.sh b/tests/compose/test-basic.sh index 796faaf5..2b0ecf3c 100755 --- a/tests/compose/test-basic.sh +++ b/tests/compose/test-basic.sh @@ -1,11 +1,40 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +# XXX: nuke this test once we fully drop non-unified core mode + +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" + +# Add a local rpm-md repo so we can mutate local test packages +treefile_append "repos" '["test-repo"]' +# test `recommends: false` (test-misc-tweaks tests the true path) +build_rpm foobar recommends foobar-rec +build_rpm foobar-rec + +echo gpgcheck=0 >> yumrepo.repo +ln "$PWD/yumrepo.repo" config/yumrepo.repo +treefile_append "packages" '["foobar"]' + +### +# MAJOR HACK ALERT; drop modular pkgs because libdnf in the non-unified path +# wants the modulemd available in the rpmmd, which neither our cache repo nor +# pool have +treefile_pyedit " +tf['packages'].remove('afterburn') +tf['packages'].remove('afterburn-dracut') +tf['packages'].remove('fedora-coreos-pinger') +tf['packages'].remove('sssd') +" +# have mercy +echo 'exclude=libnghttp2' >> config/cache.repo +build_rpm fake-libnghttp2 version 1.40.0 provides "libnghttp2.so.14()(64bit)" +### + +treefile_pyedit "tf['add-commit-metadata']['foobar'] = 'bazboo'" +treefile_pyedit "tf['add-commit-metadata']['overrideme'] = 'old var'" -prepare_compose_test "basic" # Test metadata json with objects, arrays, numbers cat > metadata.json < metadata.json < autovar.txt +ostree --repo="${repo}" cat "${treeref}" /usr/lib/tmpfiles.d/rpm-ostree-1-autovar.conf > autovar.txt # Picked this one at random as an example of something that won't likely be # converted to tmpfiles.d upstream. But if it is, we can change this test. assert_file_has_content_literal autovar.txt 'd /var/cache 0755 root root - -' @@ -33,22 +69,23 @@ assert_file_has_content_literal autovar.txt 'd /var/cache 0755 root root - -' assert_file_has_content_literal autovar.txt 'd /var/log/chrony 0755 chrony chrony - -' echo "ok autovar" -ostree --repo=${repobuild} cat ${treeref} /usr/lib/systemd/system-preset/40-rpm-ostree-auto.preset > preset.txt +ostree --repo="${repo}" cat "${treeref}" /usr/lib/systemd/system-preset/40-rpm-ostree-auto.preset > preset.txt assert_file_has_content preset.txt '^enable ostree-remount.service$' assert_file_has_content preset.txt '^enable ostree-finalize-staged.path$' -prepare_compose_test "from-yaml" python3 < bootls.txt +ostree --repo="${repo}" ls -R "${treeref}" /boot > bootls.txt cat >bootls-expected.txt < bootls.txt +ostree --repo="${repo}" ls -R "${treeref}" /usr/lib/ostree-boot > bootls.txt assert_not_file_has_content bootls.txt vmlinuz- assert_not_file_has_content bootls.txt initramfs- # And use the kver to find the kernel in /usr/lib/modules -ostree --repo=${repobuild} ls -R ${treeref} /usr/lib/modules > modules-lsr.txt +ostree --repo="${repo}" ls -R "${treeref}" /usr/lib/modules > modules-lsr.txt assert_file_has_content modules-lsr.txt '/vmlinuz$' assert_file_has_content modules-lsr.txt '/initramfs.img$' echo "ok boot location modules" diff --git a/tests/compose/test-boot-location-new.sh b/tests/compose/test-boot-location-new.sh index 4f62933d..e9416d01 100755 --- a/tests/compose/test-boot-location-new.sh +++ b/tests/compose/test-boot-location-new.sh @@ -1,25 +1,25 @@ #!/bin/bash set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "bootlocation-new" -pysetjsonmember "boot_location" '"new"' +treefile_set boot-location '"new"' runcompose echo "ok compose" # Nothing in /boot (but it should exist) -ostree --repo=${repobuild} ls -R ${treeref} /boot > bootls.txt +ostree --repo="${repo}" ls -R "${treeref}" /boot > bootls.txt cat >bootls-expected.txt < bootls.txt +ostree --repo="${repo}" ls -R "${treeref}" /usr/lib/ostree-boot > bootls.txt assert_file_has_content bootls.txt vmlinuz- assert_file_has_content bootls.txt initramfs- kver=$(grep /vmlinuz bootls.txt | sed -e 's,.*/vmlinuz-\(.*\)-[0-9a-f].*$,\1,') # And use the kver to find the kernel in /usr/lib/modules -ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/{vmlinuz,initramfs.img} >/dev/null +ostree --repo="${repo}" ls "${treeref}" "/usr/lib/modules/${kver}"/{vmlinuz,initramfs.img} >/dev/null echo "ok boot location new" diff --git a/tests/compose/test-install-langs.sh b/tests/compose/test-install-langs.sh index d2f74106..099871db 100755 --- a/tests/compose/test-install-langs.sh +++ b/tests/compose/test-install-langs.sh @@ -1,12 +1,12 @@ #!/bin/bash set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "install-langs" -pysetjsonmember "install-langs" '["fr", "fr_FR", "en_US"]' -pysetjsonmember "postprocess-script" \"$PWD/lang-test.sh\" +treefile_set "install-langs" '["fr", "fr_FR", "en_US"]' +treefile_set "postprocess-script" "'$PWD/lang-test.sh'" cat > lang-test.sh << EOF #!/bin/bash set -xeuo pipefail @@ -19,18 +19,21 @@ chmod a+x lang-test.sh runcompose echo "ok compose" -ostree --repo=${repobuild} cat ${treeref} /usr/etc/lang-test.date.txt > out.txt +ostree --repo=${repo} cat ${treeref} /usr/etc/lang-test.date.txt > out.txt assert_file_has_content out.txt 'jeu\. janv\. 1 00:00:00 UTC 1970' -ostree --repo=${repobuild} cat ${treeref} /usr/etc/lang-test.touch.txt > out.txt +ostree --repo=${repo} cat ${treeref} /usr/etc/lang-test.touch.txt > out.txt assert_file_has_content out.txt 'opérande de fichier manquant' -# check that de_DE was culled -ostree --repo=${repobuild} cat ${treeref} /usr/etc/lang-test.de.date.txt > out.txt -assert_file_has_content out.txt 'Thu Jan 1 00:00:00 UTC 1970' -ostree --repo=${repobuild} cat ${treeref} /usr/etc/lang-test.de.touch.txt > out.txt -assert_file_has_content out.txt 'missing file operand' +# XXX: this test doesn't currently work since glibc no longer obeys install-langs: +# https://github.com/coreos/fedora-coreos-config/issues/194#issuecomment-556365516 -if ostree --repo=${repobuild} ls ${treeref} /usr/bin/rpmostree-postprocess-lang-test.sh 2>err.txt; then +# # check that de_DE was culled +# ostree --repo=${repo} cat ${treeref} /usr/etc/lang-test.de.date.txt > out.txt +# assert_file_has_content out.txt 'Thu Jan 1 00:00:00 UTC 1970' +# ostree --repo=${repo} cat ${treeref} /usr/etc/lang-test.de.touch.txt > out.txt +# assert_file_has_content out.txt 'missing file operand' + +if ostree --repo=${repo} ls ${treeref} /usr/bin/rpmostree-postprocess-lang-test.sh 2>err.txt; then assert_not_reached "we failed to unlink?" fi assert_file_has_content err.txt "error: No such file or directory" diff --git a/tests/compose/test-installroot.sh b/tests/compose/test-installroot.sh index 9fd9fba6..078a0418 100755 --- a/tests/compose/test-installroot.sh +++ b/tests/compose/test-installroot.sh @@ -1,19 +1,27 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "installroot" # This is used to test postprocessing with treefile vs not -pysetjsonmember "boot_location" '"new"' -instroot_tmp=$(mktemp -d /var/tmp/rpm-ostree-instroot.XXXXXX) -rpm-ostree compose install --unified-core --repo="${repobuild}" ${treefile} ${instroot_tmp} +treefile_set "boot-location" '"new"' + +# This test is a bit of a degenerative case of the supermin abstration. We need +# to be able to interact with the compose output directly, feed it back to +# rpm-ostree, etc... So we just run whole scripts inside the VM. + +instroot_tmp=cache/instroot instroot=${instroot_tmp}/rootfs -assert_not_has_dir ${instroot}/etc +integrationconf=usr/lib/tmpfiles.d/rpm-ostree-0-integration.conf +runasroot sh -xec " +mkdir -p ${instroot_tmp} +rpm-ostree compose install ${compose_base_argv} ${treefile} ${instroot_tmp} + +! test -d ${instroot}/etc test -L ${instroot}/home -assert_has_dir ${instroot}/usr/etc +test -d ${instroot}/usr/etc # Clone the root - we'll test direct commit, as well as postprocess with # and without treefile. @@ -21,28 +29,32 @@ mv ${instroot}{,-postprocess} cp -al ${instroot}{-postprocess,-directcommit} cp -al ${instroot}{-postprocess,-postprocess-treefile} -integrationconf=usr/lib/tmpfiles.d/rpm-ostree-0-integration.conf -assert_not_has_file ${instroot}-postprocess/${integrationconf} +! test -f ${instroot}-postprocess/${integrationconf} rpm-ostree compose postprocess ${instroot}-postprocess -assert_has_file ${instroot}-postprocess/${integrationconf} -ostree --repo=${repobuild} commit -b test-directcommit --selinux-policy ${instroot}-postprocess --tree=dir=${instroot}-postprocess +test -f ${instroot}-postprocess/${integrationconf} +ostree --repo=${repo} commit -b test-directcommit --selinux-policy ${instroot}-postprocess --tree=dir=${instroot}-postprocess +" echo "ok postprocess + direct commit" +runasroot sh -xec " rpm-ostree compose postprocess ${instroot}-postprocess-treefile ${treefile} -assert_has_file ${instroot}-postprocess-treefile/${integrationconf} +test -f ${instroot}-postprocess-treefile/${integrationconf} # with treefile, no kernels in /boot ls ${instroot}-postprocess-treefile/boot > ls.txt -assert_not_file_has_content ls.txt '^vmlinuz-' +! grep '^vmlinuz-' ls.txt rm -f ls.txt +" echo "ok postprocess with treefile" testdate=$(date) -echo "${testdate}" > ${instroot}-directcommit/usr/share/rpm-ostree-composetest-split.txt -assert_not_has_file ${instroot}-directcommit/${integrationconf} -rpm-ostree compose commit --repo=${repobuild} ${treefile} ${instroot}-directcommit -ostree --repo=${repobuild} ls ${treeref} /usr/bin/bash -ostree --repo=${repobuild} cat ${treeref} /usr/share/rpm-ostree-composetest-split.txt >out.txt -assert_file_has_content_literal out.txt "${testdate}" -ostree --repo=${repobuild} cat ${treeref} /${integrationconf} +runasroot sh -xec " +echo \"${testdate}\" > ${instroot}-directcommit/usr/share/rpm-ostree-composetest-split.txt +! test -f ${instroot}-directcommit/${integrationconf} +rpm-ostree compose commit --repo=${repo} ${treefile} ${instroot}-directcommit +ostree --repo=${repo} ls ${treeref} /usr/bin/bash +ostree --repo=${repo} cat ${treeref} /usr/share/rpm-ostree-composetest-split.txt >out.txt +grep \"${testdate}\" out.txt +ostree --repo=${repo} cat ${treeref} /${integrationconf} +" echo "ok installroot" diff --git a/tests/compose/test-lockfile.sh b/tests/compose/test-lockfile.sh index 531611f2..dba59a89 100755 --- a/tests/compose/test-lockfile.sh +++ b/tests/compose/test-lockfile.sh @@ -1,48 +1,47 @@ #!/bin/bash set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "lockfile" # Add a local rpm-md repo so we can mutate local test packages -pyappendjsonmember "repos" '["test-repo"]' +treefile_append "repos" '["test-repo"]' build_rpm test-pkg-common build_rpm test-pkg requires test-pkg-common build_rpm another-test-pkg -# The test suite writes to pwd, but we need repos in composedata + +# The test suite writes to pwd, but we need repos together with the manifests # Also we need to disable gpgcheck echo gpgcheck=0 >> yumrepo.repo -ln yumrepo.repo composedata/test-repo.repo -pyappendjsonmember "packages" '["test-pkg", "another-test-pkg"]' -pysetjsonmember "documentation" 'False' -mkdir cache -# Create lockfile -runcompose --ex-write-lockfile-to=$PWD/versions.lock --cachedir $(pwd)/cache -rpm-ostree --repo=${repobuild} db list ${treeref} > test-pkg-list.txt +ln "$PWD/yumrepo.repo" config/yumrepo.repo +treefile_append "packages" '["test-pkg", "another-test-pkg"]' + +runcompose --ex-write-lockfile-to="$PWD/versions.lock" +rpm-ostree --repo=${repo} db list ${treeref} > test-pkg-list.txt assert_file_has_content test-pkg-list.txt 'test-pkg-1.0-1.x86_64' assert_file_has_content test-pkg-list.txt 'another-test-pkg-1.0-1.x86_64' echo "ok compose" assert_has_file "versions.lock" -assert_jq versions.lock \ +assert_jq "versions.lock" \ '.packages["test-pkg"].evra = "1.0-1.x86_64"' \ '.packages["test-pkg-common"].evra = "1.0-1.x86_64"' \ '.packages["another-test-pkg"].evra = "1.0-1.x86_64"' \ '.metadata.rpmmd_repos|length > 0' \ '.metadata.generated' echo "ok lockfile created" -# Read lockfile back + +# Read lockfile back (should be a no-op) build_rpm test-pkg-common version 2.0 build_rpm test-pkg version 2.0 requires test-pkg-common build_rpm another-test-pkg version 2.0 -runcompose --ex-lockfile=$PWD/versions.lock --cachedir $(pwd)/cache -echo "ok compose with lockfile" +runcompose --ex-lockfile="$PWD/versions.lock" |& tee out.txt -rpm-ostree --repo=${repobuild} db list ${treeref} > test-pkg-list.txt -assert_file_has_content test-pkg-list.txt 'test-pkg-1.0-1.x86_64' -assert_file_has_content test-pkg-list.txt 'test-pkg-common-1.0-1.x86_64' -assert_file_has_content test-pkg-list.txt 'another-test-pkg-1.0-1.x86_64' +rpm-ostree --repo=${repo} db list ${treeref} > test-pkg-list.txt +assert_file_has_content out.txt 'test-pkg-1.0-1.x86_64' +assert_file_has_content out.txt 'test-pkg-common-1.0-1.x86_64' +assert_file_has_content out.txt 'another-test-pkg-1.0-1.x86_64' echo "ok lockfile read" # now add an override and check that not specifying a digest is allowed @@ -56,11 +55,11 @@ cat > override.lock < ${treefile}.new -mv ${treefile}{.new,} -treeref="" -pysetjsonmember "machineid-compat" 'False' -cat > composedata/fedora-machineid-compat-includer.yaml <refs.txt -diff -u /dev/null refs.txt +ostree --repo="${repo}" refs > refs.txt +assert_not_file_has_content refs.txt "${treeref}" echo "ok no refs written" -ostree --repo=${repobuild} ls ${commit} /usr/etc > ls.txt -assert_not_file_has_content ls.txt 'machine-id' +commit=$(jq -r '.["ostree-commit"]' < compose.json) +ostree --repo=${repo} ls ${commit} /usr/etc > ls.txt +assert_file_has_content ls.txt 'machine-id' echo "ok machineid-compat" diff --git a/tests/compose/test-misc-tweaks.sh b/tests/compose/test-misc-tweaks.sh index 1396d85b..784299c3 100755 --- a/tests/compose/test-misc-tweaks.sh +++ b/tests/compose/test-misc-tweaks.sh @@ -1,32 +1,47 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "misc-tweaks" -# No docs, also test multi includes -cat >composedata/documentation.yaml <<'EOF' -documentation: false +# Add a local rpm-md repo so we can mutate local test packages +treefile_append "repos" '["test-repo"]' +# test `recommends: true` (test-basic[-unified] test the false path) +build_rpm foobar recommends foobar-rec +build_rpm foobar-rec + +echo gpgcheck=0 >> yumrepo.repo +ln "$PWD/yumrepo.repo" config/yumrepo.repo +# the top-level manifest doesn't have any packages, so just set it +treefile_append "packages" '["foobar"]' + +# With docs and recommends, also test multi includes +cat > config/documentation.yaml <<'EOF' +documentation: true EOF -cat > composedata/recommends.yaml <<'EOF' -recommends: false +cat > config/recommends.yaml <<'EOF' +recommends: true EOF -pysetjsonmember "include" '["documentation.yaml", "recommends.yaml"]' +treefile_append "include" '["documentation.yaml", "recommends.yaml"]' +treefile_del 'recommends' +treefile_del 'documentation' + # Note this overrides: # $ rpm -q systemd -# systemd-238-8.git0e0aa59.fc28.x86_64 +# systemd-243.4-1.fc31.x86_64 # $ rpm -qlv systemd|grep -F 'system/default.target ' # lrwxrwxrwx 1 root root 16 May 11 06:59 /usr/lib/systemd/system/default.target -> graphical.target -pysetjsonmember "default_target" '"multi-user.target"' -pysetjsonmember "units" '["tuned.service"]' +treefile_set "default-target" '"multi-user.target"' +treefile_append "units" '["zincati.service"]' +# Need this in order to test unit enablement +treefile_set "machineid-compat" "True" # And test adding/removing files -pysetjsonmember "add-files" '[["foo.txt", "/usr/etc/foo.txt"], - ["baz.txt", "/usr/share/baz.txt"], - ["bar.txt", "/etc/bar.txt"]]' -pysetjsonmember "postprocess-script" \"$PWD/postprocess.sh\" -pysetjsonmember "postprocess" '["""#!/bin/bash +treefile_append "add-files" '[["foo.txt", "/usr/etc/foo.txt"], + ["baz.txt", "/usr/share/baz.txt"], + ["bar.txt", "/etc/bar.txt"]]' +treefile_set "postprocess-script" \"$PWD/postprocess.sh\" +treefile_set "postprocess" '["""#!/bin/bash touch /usr/share/postprocess-testing""", """#!/bin/bash set -xeuo pipefail @@ -43,16 +58,16 @@ cp -a /usr/etc/foo.txt /usr/share/misc-tweaks-foo.txt EOF chmod a+x postprocess.sh -pysetjsonmember "remove-files" '["etc/hosts"]' -pysetjsonmember "remove-from-packages" '[["setup", "/etc/hosts\..*"]]' +treefile_set "remove-files" '["etc/hosts"]' +treefile_set "remove-from-packages" '[["setup", "/etc/hosts\..*"]]' rnd=$RANDOM -echo $rnd > composedata/foo.txt -echo bar > composedata/bar.txt -echo baz > composedata/baz.txt -# Test tmp-is-dir -pysetjsonmember "tmp-is-dir" 'True' +echo $rnd > config/foo.txt +echo bar > config/bar.txt +echo baz > config/baz.txt +# Test tmp-is-dir False +treefile_set "tmp-is-dir" 'False' -new_treefile=composedata/fedora-misc-tweaks-includer.yaml +new_treefile=config/fedora-misc-tweaks-includer.yaml cat > ${new_treefile} < tmp/usr/bin/sometestbinary-${x} - chmod a+x tmp/usr/bin/sometestbinary-${x} - echo sometestdata${x} > tmp/usr/share/sometestdata-${x} - echo sometestdata-subdir-${x} > tmp/usr/share/testsubdir-${x}/test - ostree --repo="${repobuild}" commit --consume --no-xattrs --owner-uid=0 --owner-gid=0 -b testlayer-${x} --tree=dir=tmp + rm tmp/rootfs/usr -rf + mkdir -p tmp/rootfs/usr/{bin,share} + mkdir tmp/rootfs/usr/share/testsubdir-${x} + echo sometest${x} > tmp/rootfs/usr/bin/sometestbinary-${x} + chmod a+x tmp/rootfs/usr/bin/sometestbinary-${x} + echo sometestdata${x} > tmp/rootfs/usr/share/sometestdata-${x} + echo sometestdata-subdir-${x} > tmp/rootfs/usr/share/testsubdir-${x}/test + ostree --repo="${repo}" commit --consume --no-xattrs --owner-uid=0 --owner-gid=0 -b testlayer-${x} --tree=dir=tmp/rootfs done -rm tmp/usr -rf -mkdir -p tmp/usr/{share/info,bin} -echo sweet new ls binary > tmp/usr/bin/ls -ostree --repo="${repobuild}" commit --consume --no-xattrs --owner-uid=0 --owner-gid=0 -b testoverride-1 --tree=dir=tmp +rm tmp/rootfs/usr -rf + +mkdir -p tmp/rootfs/usr/{share/info,bin} +echo sweet new ls binary > tmp/rootfs/usr/bin/ls +ostree --repo="${repo}" commit --consume --no-xattrs --owner-uid=0 --owner-gid=0 -b testoverride-1 --tree=dir=tmp/rootfs cat >> ${new_treefile} < manpages.txt -assert_not_file_has_content manpages.txt man5/ostree.repo.5 -echo "ok no manpages" +# Tests for docs +ostree --repo=${repo} ls -R ${treeref} /usr/share/man > manpages.txt +assert_file_has_content manpages.txt man5/ostree.repo.5 +echo "ok manpages" # Tests for units -ostree --repo=${repobuild} ls ${treeref} \ +ostree --repo=${repo} ls ${treeref} \ /usr/lib/systemd/system/default.target > out.txt assert_file_has_content out.txt '-> .*multi-user\.target' echo "ok default target" -ostree --repo=${repobuild} ls ${treeref} \ +ostree --repo=${repo} ls ${treeref} \ /usr/etc/systemd/system/multi-user.target.wants > out.txt -assert_file_has_content out.txt '-> .*/tuned.service' +assert_file_has_content out.txt '-> .*/zincati.service' echo "ok enable units" # Tests for files -ostree --repo=${repobuild} cat ${treeref} /usr/etc/foo.txt > out.txt +ostree --repo=${repo} cat ${treeref} /usr/etc/foo.txt > out.txt assert_file_has_content out.txt $rnd -ostree --repo=${repobuild} cat ${treeref} /usr/etc/bar.txt > out.txt +ostree --repo=${repo} cat ${treeref} /usr/etc/bar.txt > out.txt assert_file_has_content out.txt bar -ostree --repo=${repobuild} cat ${treeref} /usr/share/baz.txt > out.txt +ostree --repo=${repo} cat ${treeref} /usr/share/baz.txt > out.txt assert_file_has_content out.txt baz # https://github.com/projectatomic/rpm-ostree/pull/997 -ostree --repo=${repobuild} cat ${treeref} /usr/share/misc-tweaks-foo.txt > out.txt +ostree --repo=${repo} cat ${treeref} /usr/share/misc-tweaks-foo.txt > out.txt assert_file_has_content out.txt $rnd echo "ok add-files" -ostree --repo=${repobuild} ls ${treeref} /usr/etc > out.txt +ostree --repo=${repo} ls ${treeref} /usr/etc > out.txt assert_not_file_has_content out.txt '/usr/etc/hosts$' echo "ok remove-files" -ostree --repo=${repobuild} ls ${treeref} /usr/etc > out.txt +ostree --repo=${repo} ls ${treeref} /usr/etc > out.txt assert_not_file_has_content out.txt '/usr/etc/hosts\.allow$' assert_not_file_has_content out.txt '/usr/etc/hosts\.deny$' echo "ok remove-from-packages" # https://github.com/projectatomic/rpm-ostree/issues/669 -ostree --repo=${repobuild} ls ${treeref} /tmp > ls.txt -assert_file_has_content ls.txt 'd01777 0 0 0 /tmp' +ostree --repo=${repo} ls ${treeref} /tmp > ls.txt +assert_file_has_content ls.txt 'l00777 0 0 0 /tmp -> sysroot/tmp' echo "ok /tmp" -ostree --repo=${repobuild} show ${treeref} \ - --print-metadata-key rpmostree.rpmdb.pkglist > pkglist.txt -# This is currently a Recommends: package. If you change this, please -# also change the corresponding test in libbasic-test.sh. -assert_file_has_content_literal pkglist.txt 'systemd-' -assert_not_file_has_content pkglist.txt 'systemd-bootchart' +rpm-ostree db list --repo=${repo} ${treeref} > pkglist.txt +assert_file_has_content_literal pkglist.txt 'foobar' +assert_file_has_content_literal pkglist.txt 'foobar-rec' echo "ok recommends" # Test overlays/overrides for x in $(seq 3); do - ostree --repo=${repobuild} cat ${treeref} /usr/bin/sometestbinary-${x} > t + ostree --repo=${repo} cat ${treeref} /usr/bin/sometestbinary-${x} > t assert_file_has_content t "sometest${x}" - ostree --repo=${repobuild} cat ${treeref} /usr/share/testsubdir-${x}/test > t + ostree --repo=${repo} cat ${treeref} /usr/share/testsubdir-${x}/test > t assert_file_has_content t sometestdata-subdir-${x} done -ostree --repo=${repobuild} cat ${treeref} /usr/bin/ls > ls.txt +ostree --repo=${repo} cat ${treeref} /usr/bin/ls > ls.txt assert_file_has_content ls.txt '^sweet new ls binary$' echo "ok layers" # Check that add-files with bad paths are rejected -prepare_compose_test "add-files-failure" -pysetjsonmember "add-files" '[["foo.txt", "/var/lib/foo.txt"]]' +treefile_append "add-files" '[["foo.txt", "/var/lib/foo.txt"]]' -# Do the compose ourselves since set -e doesn't work in function calls in if -rm ${compose_workdir} -rf -mkdir ${test_tmpdir}/workdir -if rpm-ostree compose tree ${compose_base_argv} ${treefile} |& tee err.txt; then - assert_not_reached err.txt "Successfully composed with add-files for /var/lib?" +if runcompose |& tee err.txt; then + assert_not_reached "Successfully composed with add-files for /var/lib?" fi assert_file_has_content_literal err.txt "Unsupported path in add-files: /var" echo "ok bad add-files" diff --git a/tests/compose/test-mutate-os-release.sh b/tests/compose/test-mutate-os-release.sh index da1791dd..50063982 100755 --- a/tests/compose/test-mutate-os-release.sh +++ b/tests/compose/test-mutate-os-release.sh @@ -1,60 +1,61 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh -releasever=29 +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" + +releasever=31 + +# make sure we clear out postprocess scripts, which might be using os-release +treefile_set "postprocess" '[]' # specifying the key but neither automatic_version_prefix nor # --add-metadata-string should cause no mutation - -prepare_compose_test "mutate-os-release-none" -pysetjsonmember "mutate-os-release" '"'${releasever}'"' +treefile_set "mutate-os-release" '"'${releasever}'"' +treefile_del 'automatic-version-prefix' runcompose echo "ok compose (none)" -ostree --repo=${repobuild} cat ${treeref} \ - /usr/lib/os.release.d/os-release-atomichost > os-release.prop +ostree --repo=${repo} cat ${treeref} \ + /usr/lib/os-release > os-release.prop assert_file_has_content os-release.prop VERSION_ID=${releasever} assert_not_file_has_content os-release.prop OSTREE_VERSION= -assert_file_has_content os-release.prop 'VERSION="'${releasever}' (Atomic ' +assert_file_has_content os-release.prop 'VERSION="'${releasever}' (CoreOS' echo "ok mutate-os-release-none" # make sure --add-metadata-string has precedence and works with # mutate-os-release -prepare_compose_test "mutate-os-release-cli" -pysetjsonmember "automatic_version_prefix" '"'${releasever}'.555"' -pysetjsonmember "mutate-os-release" '"'${releasever}'"' +treefile_set "automatic-version-prefix" '"'${releasever}'.555"' +treefile_set "mutate-os-release" '"'${releasever}'"' runcompose --add-metadata-string=version=${releasever}.444 echo "ok compose (cli)" -ostree --repo=${repobuild} cat ${treeref} \ - /usr/lib/os.release.d/os-release-atomichost > os-release.prop +ostree --repo=${repo} cat ${treeref} \ + /usr/lib/os-release > os-release.prop # VERSION_ID *shouldn't* change # (https://github.com/projectatomic/rpm-ostree/pull/433) assert_file_has_content os-release.prop VERSION_ID=${releasever} assert_file_has_content os-release.prop OSTREE_VERSION=\'${releasever}.444\' -assert_file_has_content os-release.prop 'VERSION="'${releasever}'\.444 (Atomic ' +assert_file_has_content os-release.prop 'VERSION="'${releasever}'\.444 (CoreOS' echo "ok mutate-os-release-cli" # make sure automatic_version_prefix works -prepare_compose_test "mutate-os-release-auto" -pysetjsonmember "automatic_version_prefix" '"'${releasever}'.555"' -pysetjsonmember "mutate-os-release" '"'${releasever}'"' -runcompose +treefile_set "automatic-version-prefix" '"'${releasever}'.555"' +treefile_set "mutate-os-release" '"'${releasever}'"' +runcompose --force-nocache echo "ok compose (auto)" -ostree --repo=${repobuild} cat ${treeref} \ - /usr/lib/os.release.d/os-release-atomichost > os-release.prop +ostree --repo=${repo} cat ${treeref} \ + /usr/lib/os-release > os-release.prop # VERSION_ID *shouldn't* change # (https://github.com/projectatomic/rpm-ostree/pull/433) assert_file_has_content os-release.prop VERSION_ID=${releasever} assert_file_has_content os-release.prop OSTREE_VERSION=\'${releasever}.555\' -assert_file_has_content os-release.prop 'VERSION="'${releasever}'\.555 (Atomic ' +assert_file_has_content os-release.prop 'VERSION="'${releasever}'\.555 (CoreOS' echo "ok mutate-os-release (auto)" diff --git a/tests/compose/test-rojig-e2e.sh b/tests/compose/test-rojig-e2e.sh index 7a53bbd1..7b340519 100755 --- a/tests/compose/test-rojig-e2e.sh +++ b/tests/compose/test-rojig-e2e.sh @@ -1,47 +1,45 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh -. ${dn}/../common/libtest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "rojig" # Add a local rpm-md repo so we can mutate local test packages -pyappendjsonmember "repos" '["test-repo"]' +treefile_append "repos" '["test-repo"]' build_rpm test-pkg \ files "/usr/bin/test-pkg" \ install "mkdir -p %{buildroot}/usr/bin && echo localpkg data > %{buildroot}/usr/bin/test-pkg" -# The test suite writes to pwd, but we need repos in composedata + +# The test suite writes to pwd, but we need repos in config # Also we need to disable gpgcheck echo gpgcheck=0 >> yumrepo.repo -ln yumrepo.repo composedata/test-repo.repo -pyappendjsonmember "packages" '["test-pkg"]' -pysetjsonmember "documentation" 'False' -# Need unified core for this, as well as a cachedir -mkdir cache -runcompose --unified-core --cachedir $(pwd)/cache --add-metadata-string version=42.0 -npkgs=$(rpm-ostree --repo=${repobuild} db list ${treeref} |grep -v '^ostree commit' | wc -l) +ln yumrepo.repo config/test-repo.repo +treefile_append "packages" '["test-pkg"]' +treefile_set "documentation" 'False' + +runcompose --add-metadata-string version=42.0 +npkgs=$(rpm-ostree --repo=${repo} db list ${treeref} |grep -v '^ostree commit' | wc -l) echo "npkgs=${npkgs}" -rpm-ostree --repo=${repobuild} db list ${treeref} test-pkg >test-pkg-list.txt +rpm-ostree --repo=${repo} db list ${treeref} test-pkg >test-pkg-list.txt assert_file_has_content test-pkg-list.txt 'test-pkg-1.0-1.x86_64' -rev=$(ostree --repo=${repobuild} rev-parse ${treeref}) +rev=$(ostree --repo=${repo} rev-parse ${treeref}) mkdir rojig-output do_commit2rojig() { targetrev=$1 echo "$(date): starting commit2rojig" - rpm-ostree ex commit2rojig --repo=repo-build --pkgcache-repo cache/pkgcache-repo ${targetrev} ${treefile} $(pwd)/rojig-output + runasroot rpm-ostree ex commit2rojig --repo=${repo} --pkgcache-repo cache/pkgcache-repo ${targetrev} ${treefile} $(pwd)/rojig-output (cd rojig-output && createrepo_c .) echo "$(date): finished commit2rojig" } do_commit2rojig ${rev} -test -f rojig-output/x86_64/fedora-atomic-host-42.0-1.fc29.x86_64.rpm +test -f rojig-output/x86_64/fedora-coreos-42.0-1.fc31.x86_64.rpm ostree --repo=rojig-unpack-repo init --mode=bare-user echo 'fsync=false' >> rojig-unpack-repo/config -# Technically this isn't part of composedata but eh -cat > composedata/rojig-test.repo < config/rojig-test.repo < %{buildroot}/usr/bin/test-newpkg" -pyappendjsonmember "packages" '["test-newpkg"]' -runcompose --unified-core --cachedir $(pwd)/cache --add-metadata-string version=42.1 -newrev=$(ostree --repo=${repobuild} rev-parse ${treeref}) -rpm-ostree --repo=${repobuild} db list ${treeref} test-newpkg >test-newpkg-list.txt +treefile_append "packages" '["test-newpkg"]' +runcompose --add-metadata-string version=42.1 +newrev=$(ostree --repo=${repo} rev-parse ${treeref}) +rpm-ostree --repo=${repo} db list ${treeref} test-newpkg >test-newpkg-list.txt assert_file_has_content test-newpkg-list.txt 'test-newpkg-1.0-1.x86_64' # Rojig version 42.1 do_commit2rojig ${newrev} -path=rojig-output/x86_64/fedora-atomic-host-42.1-1.fc29.x86_64.rpm +path=rojig-output/x86_64/fedora-coreos-42.1-1.fc31.x86_64.rpm rpm -qp --requires ${path} > requires.txt assert_file_has_content requires.txt 'glibc(.*) = ' assert_file_has_content requires.txt 'systemd(.*) = ' @@ -106,12 +104,12 @@ assert_file_has_content rojig-refs.txt 'rpmostree/rojig/test-pkg/1.1-1.x86__64' echo "ok rojig ♲📦 update!" # Add all docs to test https://github.com/projectatomic/rpm-ostree/issues/1197 -pysetjsonmember "documentation" 'True' -runcompose --unified-core --cachedir $(pwd)/cache --add-metadata-string version=42.2 -newrev=$(ostree --repo=${repobuild} rev-parse ${treeref}) +treefile_set "documentation" 'True' +runcompose --add-metadata-string version=42.2 +newrev=$(ostree --repo=${repo} rev-parse ${treeref}) do_commit2rojig ${newrev} find rojig-output -name '*.rpm' | tee rpms.txt -assert_file_has_content rpms.txt 'fedora-atomic-host-42.2.*x86_64' +assert_file_has_content rpms.txt 'fedora-coreos-42.2.*x86_64' do_rojig2commit # Not every package has docs, but there are going to need to be changes assert_file_has_content rojig2commit-out.txt '[1-9][0-9]*/[1-9][0-9]* packages to import ([1-9][0-9]* changed)' diff --git a/tests/compose/test-rojig-pure.sh b/tests/compose/test-rojig-pure.sh index 3f5d7564..88937c4b 100755 --- a/tests/compose/test-rojig-pure.sh +++ b/tests/compose/test-rojig-pure.sh @@ -1,33 +1,31 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh -. ${dn}/../common/libtest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "rojig-pure" -pysetjsonmember "automatic_version_prefix" '"42"' -mkdir cache +treefile_set "automatic-version-prefix" '"42"' +treefile_set "documentation" 'True' mkdir rojig-repo runcompose() { (cd rojig-repo && createrepo_c .) && \ rm -f treecompose.json && \ - rpm-ostree compose rojig --write-composejson-to $(pwd)/treecompose.json --cachedir=$(pwd)/cache ${treefile} $(pwd)/rojig-repo "$@" && \ + runasroot rpm-ostree compose rojig --write-composejson-to $(pwd)/treecompose.json --cachedir=$(pwd)/cache ${treefile} $(pwd)/rojig-repo "$@" && \ (cd rojig-repo && createrepo_c .) } runcompose test -f treecompose.json -test -f rojig-repo/x86_64/fedora-atomic-host-42-1.fc29.x86_64.rpm +test -f rojig-repo/x86_64/fedora-coreos-42-1.fc31.x86_64.rpm echo "ok rojig ♲📦 initial" runcompose test '!' -f treecompose.json echo "ok rojig no changes" -pysetjsonmember "documentation" 'False' +treefile_set "documentation" 'False' runcompose test -f treecompose.json -test -f rojig-repo/x86_64/fedora-atomic-host-42.1-1.fc29.x86_64.rpm +test -f rojig-repo/x86_64/fedora-coreos-42.1-1.fc31.x86_64.rpm echo "ok rojig dropped docs" diff --git a/tests/compose/test-write-commitid.sh b/tests/compose/test-write-commitid.sh index b066db74..1e6c9736 100755 --- a/tests/compose/test-write-commitid.sh +++ b/tests/compose/test-write-commitid.sh @@ -1,28 +1,24 @@ #!/bin/bash - set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -. ${dn}/libcomposetest.sh +dn=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=libcomposetest.sh +. "${dn}/libcomposetest.sh" -prepare_compose_test "write-commitid" -treeref="" runcompose --write-commitid-to $(pwd)/commitid.txt wc -c < commitid.txt > wc.txt assert_file_has_content_literal wc.txt 64 echo "ok compose" # --write-commitid-to should not set the ref -if ostree --repo=${repobuild} rev-parse ${treeref}; then - fatal "Found ${treeref} ?" -fi +ostree --repo=${repo} refs > refs.txt +assert_file_empty refs.txt echo "ok ref not written" commitid_txt=$(cat commitid.txt) -assert_streq "${commit}" "${commitid_txt}" +assert_streq "$(jq -r '.["ostree-commit"]' < compose.json)" "${commitid_txt}" # And verify we have other keys for key in ostree-version rpm-ostree-inputhash ostree-content-bytes-written; do - jq -r '.["'${key}'"]' ${composejson} >/dev/null + jq -r '.["'${key}'"]' compose.json >/dev/null done - echo "ok composejson" diff --git a/tests/composedata/fedora-base.json b/tests/composedata/fedora-base.json deleted file mode 100644 index 9b7f5703..00000000 --- a/tests/composedata/fedora-base.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "rojig": { - "name": "fedora-atomic-host", - "license": "MIT", - "summary": "Fedora Atomic Host" - }, - "releasever": "29", - - "repos": ["fedora", "updates"], - - "packages": ["kernel", "nss-altfiles", "systemd", "ostree", "selinux-policy-targeted", "chrony", - "tuned", "iputils", "fedora-release-atomichost", "docker", "container-selinux"], - - "packages-aarch64": ["grub2-efi", "ostree-grub2", - "efibootmgr", "shim"], - - "packages-armhfp": ["extlinux-bootloader"], - - "packages-ppc64": ["grub2", "ostree-grub2"], - - "packages-ppc64le": ["grub2", "ostree-grub2"], - - "packages-x86_64": ["grub2", "grub2-efi", "ostree-grub2", - "efibootmgr", "shim"], - - "ignore-removed-users": ["root"], - "ignore-removed-groups": ["root"], - "check-passwd": { "type": "file", "filename": "passwd" }, - "check-groups": { "type": "file", "filename": "group" }, - "add-commit-metadata": { - "foobar": "bazboo", - "overrideme": "old var" - } -} diff --git a/tests/composedata/group b/tests/composedata/group deleted file mode 100644 index 0a2dab07..00000000 --- a/tests/composedata/group +++ /dev/null @@ -1,51 +0,0 @@ -root:x:0: -bin:x:1: -daemon:x:2: -sys:x:3: -adm:x:4: -tty:x:5: -disk:x:6: -lp:x:7: -mem:x:8: -kmem:x:9: -wheel:x:10: -cdrom:x:11: -mail:x:12: -man:x:15: -dialout:x:18: -floppy:x:19: -games:x:20: -tape:x:30: -video:x:39: -ftp:x:50: -lock:x:54: -audio:x:63: -nobody:x:99: -users:x:100: -utmp:x:22: -utempter:x:35: -ssh_keys:x:999: -systemd-journal:x:190: -dbus:x:81: -polkitd:x:998: -etcd:x:997: -dip:x:40: -cgred:x:996: -tss:x:59: -avahi-autoipd:x:170: -rpc:x:32: -sssd:x:993: -dockerroot:x:986: -rpcuser:x:29: -nfsnobody:x:65534: -kube:x:994: -sshd:x:74: -chrony:x:992: -tcpdump:x:72: -ceph:x:167: -input:x:995: -systemd-timesync:x:991: -systemd-network:x:990: -systemd-resolve:x:989: -systemd-bus-proxy:x:988: -cockpit-ws:x:987: diff --git a/tests/composedata/passwd b/tests/composedata/passwd deleted file mode 100644 index b05ebdbf..00000000 --- a/tests/composedata/passwd +++ /dev/null @@ -1,33 +0,0 @@ -root:x:0:0:root:/root:/bin/bash -bin:x:1:1:bin:/bin:/sbin/nologin -daemon:x:2:2:daemon:/sbin:/sbin/nologin -adm:x:3:4:adm:/var/adm:/sbin/nologin -lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin -sync:x:5:0:sync:/sbin:/bin/sync -shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown -halt:x:7:0:halt:/sbin:/sbin/halt -mail:x:8:12:mail:/var/spool/mail:/sbin/nologin -operator:x:11:0:operator:/root:/sbin/nologin -games:x:12:100:games:/usr/games:/sbin/nologin -ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin -nobody:x:99:99:Nobody:/:/sbin/nologin -dbus:x:81:81:System message bus:/:/sbin/nologin -polkitd:x:999:998:User for polkitd:/:/sbin/nologin -etcd:x:998:997:etcd user:/var/lib/etcd:/sbin/nologin -tss:x:59:59:Account used by the trousers package to sandbox the tcsd daemon:/dev/null:/sbin/nologin -avahi-autoipd:x:170:170:Avahi IPv4LL Stack:/var/lib/avahi-autoipd:/sbin/nologin -rpc:x:32:32:Rpcbind Daemon:/var/lib/rpcbind:/sbin/nologin -sssd:x:995:993:User for sssd:/:/sbin/nologin -dockerroot:x:997:986:Docker User:/var/lib/docker:/sbin/nologin -rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin -nfsnobody:x:65534:65534:Anonymous NFS User:/var/lib/nfs:/sbin/nologin -kube:x:996:994:Kubernetes user:/:/sbin/nologin -sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin -chrony:x:994:992::/var/lib/chrony:/sbin/nologin -tcpdump:x:72:72::/:/sbin/nologin -ceph:x:167:167:Ceph daemons:/var/lib/ceph:/sbin/nologin -systemd-timesync:x:993:991:systemd Time Synchronization:/:/sbin/nologin -systemd-network:x:991:990:systemd Network Management:/:/sbin/nologin -systemd-resolve:x:990:989:systemd Resolver:/:/sbin/nologin -systemd-bus-proxy:x:989:988:systemd Bus Proxy:/:/sbin/nologin -cockpit-ws:x:988:987:User for cockpit-ws:/:/sbin/nologin diff --git a/tests/composedata/treecompose-post.sh b/tests/composedata/treecompose-post.sh deleted file mode 100755 index 5bb70341..00000000 --- a/tests/composedata/treecompose-post.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -xeuo pipefail - -# Work around https://bugzilla.redhat.com/show_bug.cgi?id=1265295 -echo 'Storage=persistent' >> /etc/systemd/journald.conf - -# Work around https://github.com/systemd/systemd/issues/4082 -find /usr/lib/systemd/system/ -type f -exec sed -i -e '/^PrivateTmp=/d' -e '/^Protect\(Home\|System\)=/d' {} \; diff --git a/tests/vmcheck.sh b/tests/vmcheck.sh index 3f5f6d74..bd39e289 100755 --- a/tests/vmcheck.sh +++ b/tests/vmcheck.sh @@ -6,55 +6,21 @@ topsrcdir=$(cd "$dn/.." && pwd) commondir=$(cd "$dn/common" && pwd) export topsrcdir commondir -# https://github.com/coreos/coreos-assembler/pull/632 -ncpus() { - if ! grep -q kubepods /proc/1/cgroup; then - # this might be a developer laptop; leave one cpu free to be nice - echo $(($(nproc) - 1)) - return 0 - fi +# shellcheck source=common/libtest-core.sh +. "${commondir}/libtest-core.sh" - quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us) - period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us) - if [[ ${quota} != -1 ]] && [[ ${period} -gt 0 ]]; then - echo $(("${quota}" / "${period}")) - fi +read -r -a tests <<< "$(filter_tests "${topsrcdir}/tests/vmcheck")" +if [ ${#tests[*]} -eq 0 ]; then + echo "No tests selected; mistyped filter?" + exit 0 +fi - # just fallback to 1 - echo 1 -} +JOBS=${JOBS:-$(ncpus)} -# Just match 1:1 the number of processing units available. Ideally, we'd also -# cap based on memory available to us, but that's notoriously difficult to do -# for containers (see: -# https://fabiokung.com/2014/03/13/memory-inside-linux-containers/). We make an -# assumption here that we have at least 1G of RAM we can use per CPU available -# to us. -nhosts=${NHOSTS:-$(ncpus)} - -nselected=0 -ntotal=0 -tests=() -for tf in $(find "${topsrcdir}/tests/vmcheck/" -name 'test-*.sh' | sort); do - ntotal=$((ntotal + 1)) - - tfbn=$(basename "$tf" .sh) - tfbn=" ${tfbn#test-} " - if [ -n "${TESTS+ }" ]; then - if [[ " $TESTS " != *$tfbn* ]]; then - continue - fi - fi - - nselected=$((nselected + 1)) - tests+=(${tfbn}) -done - -echo "Running ${nselected} out of ${ntotal} tests ${nhosts} at a time" +echo "Running ${#tests[*]} tests ${JOBS} at a time" outputdir="${topsrcdir}/vmcheck-logs" echo "Test results outputting to ${outputdir}/" -if [ "${#tests[*]}" -gt 0 ]; then - echo -n "${tests[*]}" | parallel -d' ' -j "${nhosts}" --line-buffer \ - "${topsrcdir}/tests/vmcheck/runtest.sh" "${outputdir}" -fi + +echo -n "${tests[*]}" | parallel -d' ' -j "${JOBS}" --line-buffer \ + "${topsrcdir}/tests/vmcheck/runtest.sh" "${outputdir}" diff --git a/tests/vmcheck/runtest.sh b/tests/vmcheck/runtest.sh index a6c1155c..7479ba56 100755 --- a/tests/vmcheck/runtest.sh +++ b/tests/vmcheck/runtest.sh @@ -9,7 +9,7 @@ outputdir=$1; shift testname=$1; shift outputdir="${outputdir}/${testname}" -rm -rf ${outputdir}/* +rm -rf "${outputdir:?}"/* mkdir -p "${outputdir}" # keep original stdout around; this propagates to the terminal @@ -26,29 +26,29 @@ if [ -n "${V:-}" ]; then setpriv --pdeathsig SIGKILL -- tail -f "${outputdir}/output.log" >&3 & fi +echo "EXEC: ${testname}" >&3 + # this will cause libtest.sh to allocate a tmpdir and cd to it export VMTESTS=1 -echo "EXEC: ${testname}" >&3 +# shellcheck source=../common/libtest.sh disable=2154 +. "${commondir}/libtest.sh" -runtest() { - . ${commondir}/libtest.sh - . ${commondir}/libvm.sh - vm_kola_spawn "${outputdir}/kola" - "${topsrcdir}/tests/vmcheck/test-${testname}.sh" -} +# shellcheck source=../common/libvm.sh +. "${commondir}/libvm.sh" -if runtest; then +if vm_kola_spawn "${outputdir}/kola" && \ + "${topsrcdir}/tests/vmcheck/test-${testname}.sh"; then echo "PASS: ${testname}" >&3 else echo "FAIL: ${testname}" >&3 if [ -z "${V:-}" ]; then - tail -n10 "${outputdir}/output.log" | sed "s/^/ ${testname}: /g" >&3 + tail -n20 "${outputdir}/output.log" | sed "s/^/ ${testname}: /g" >&3 fi if [ -n "${VMCHECK_DEBUG:-}" ]; then echo "--- VMCHECK_DEBUG ---" >&3 - echo "To try SSH:" "SSH_AUTH_SOCK=$(realpath ${SSH_AUTH_SOCK})" ${SSH:-} >&3 + echo "To try SSH:" "SSH_AUTH_SOCK=$(realpath "${SSH_AUTH_SOCK}") ${SSH:-}" >&3 echo "Sleeping..." >&3 sleep infinity fi