rpm-ostree/tests/compose-tests/test-rojig-e2e.sh

120 lines
5.0 KiB
Bash
Raw Normal View History

#!/bin/bash
set -xeuo pipefail
dn=$(cd $(dirname $0) && pwd)
. ${dn}/libcomposetest.sh
. ${dn}/../common/libtest.sh
prepare_compose_test "rojig"
# Add a local rpm-md repo so we can mutate local test packages
pyappendjsonmember "repos" '["test-repo"]'
build_rpm test-pkg \
files "/usr/bin/test-pkg" \
install "mkdir -p %{buildroot}/usr/bin && echo localpkg data > %{buildroot}/usr/bin/test-pkg"
# The test suite writes to pwd, but we need repos in composedata
# Also we need to disable gpgcheck
echo gpgcheck=0 >> yumrepo.repo
ln yumrepo.repo composedata/test-repo.repo
pyappendjsonmember "packages" '["test-pkg"]'
jigdo V5: Use number of objects as cache invalidation trigger Changes in a server-side tree can cause the need for clients to import different objects from packages. For example, turning on documentation. Another more subtle case is where an object might "move" from package A to B by being deleted from A - then the jigdo build process will pick the B version. We need a "cache validation key"; a way for the server to tell the client that the objects it should import from the package have changed. Initially I was thinking of using the libostree "content hash" but that would be awkward as we'd have to do an import on the server side too. After more consideration I realized a simple *count* of the number of objects actually works, because (as I note in a comment) changing a file in the tree will result in it ending up in the jigdoRPM (and count as a deletion). And obviously adding or removing objects changes the count too. In fact we could have done this *without* breaking the format by just having the client start recording the number of xattr entries, but this adds greater flexibility down the line since we can in theory change how we do cache invalidation if we *really* need to (but at the cost of triggering clients to redownload packages). Note the client logic got moved around as now we need to parse all the xattrs before we decide what packages to download. My test case here is turning on docs - I noticed this actually affects *every* package which was surprising to me; I expected at least some packages wouldn't have docs. I'll double check this. It'd be good to have a "moving object" case too which I may look at. Closes: https://github.com/projectatomic/rpm-ostree/issues/1197 Closes: #1256 Approved by: jlebon
2018-02-15 23:07:39 +03:00
pysetjsonmember "documentation" 'False'
# Need unified core for this, as well as a cachedir
mkdir cache
runcompose --unified-core --cachedir $(pwd)/cache --add-metadata-string version=42.0
npkgs=$(rpm-ostree --repo=${repobuild} db list ${treeref} |grep -v '^ostree commit' | wc -l)
echo "npkgs=${npkgs}"
rpm-ostree --repo=${repobuild} db list ${treeref} test-pkg >test-pkg-list.txt
assert_file_has_content test-pkg-list.txt 'test-pkg-1.0-1.x86_64'
rev=$(ostree --repo=${repobuild} rev-parse ${treeref})
mkdir rojig-output
do_commit2rojig() {
targetrev=$1
echo "$(date): starting commit2rojig"
rpm-ostree ex commit2rojig --repo=repo-build --pkgcache-repo cache/pkgcache-repo ${targetrev} $(pwd)/composedata/fedora-atomic-host-oirpm.spec $(pwd)/rojig-output
(cd rojig-output && createrepo_c .)
echo "$(date): finished commit2rojig"
}
do_commit2rojig ${rev}
test -f rojig-output/x86_64/fedora-atomic-host-42.0-1.fc28.x86_64.rpm
ostree --repo=rojig-unpack-repo init --mode=bare-user
echo 'fsync=false' >> rojig-unpack-repo/config
# Technically this isn't part of composedata but eh
cat > composedata/rojig-test.repo <<eof
[rojig-test]
baseurl=file://$(pwd)/rojig-output
enabled=1
gpgcheck=0
eof
do_rojig2commit() {
echo "$(date): starting rojig2commit"
rpm-ostree ex rojig2commit -d $(pwd)/composedata -e fedora-local -e test-repo -e rojig-test --repo=rojig-unpack-repo rojig-test:fedora-atomic-host | tee rojig2commit-out.txt
echo "$(date): finished rojig2commit"
}
do_rojig2commit
# there will generally be pkgs not in the rojig set, but let's at least assert it's > 0
assert_file_has_content rojig2commit-out.txt ${npkgs}/${npkgs}' packages to import'
ostree --repo=rojig-unpack-repo rev-parse ${rev}
echo "$(date): starting fsck"
ostree --repo=rojig-unpack-repo fsck
echo "$(date): finished fsck"
ostree --repo=rojig-unpack-repo refs > rojig-refs.txt
assert_file_has_content rojig-refs.txt 'rpmostree/rojig/test-pkg/1.0-1.x86__64'
echo "ok rojig ♲📦 fresh assembly"
origrev=${rev}
unset rev
# Update test-pkg
build_rpm test-pkg \
version 1.1 \
files "/usr/bin/test-pkg" \
install "mkdir -p %{buildroot}/usr/bin && echo localpkg data 1.1 > %{buildroot}/usr/bin/test-pkg"
# Also add an entirely new package
build_rpm test-newpkg \
files "/usr/bin/test-newpkg" \
install "mkdir -p %{buildroot}/usr/bin && echo new localpkg data > %{buildroot}/usr/bin/test-newpkg"
pyappendjsonmember "packages" '["test-newpkg"]'
runcompose --unified-core --cachedir $(pwd)/cache --add-metadata-string version=42.1
newrev=$(ostree --repo=${repobuild} rev-parse ${treeref})
rpm-ostree --repo=${repobuild} db list ${treeref} test-newpkg >test-newpkg-list.txt
assert_file_has_content test-newpkg-list.txt 'test-newpkg-1.0-1.x86_64'
# Rojig version 42.1
do_commit2rojig ${newrev}
path=rojig-output/x86_64/fedora-atomic-host-42.1-1.fc28.x86_64.rpm
rpm -qp --requires ${path} > requires.txt
jigdo: V4: Use archful provides for jigdoRPM Requires When I tried to use my WIP client patches to do: `rpm-ostree rebase rojig://fahc:fedora-atomic-host`, I got a missing file object which turned out to be the client importing the i686 RPMs. This was passing in the test suite because we don't mirror i686 of course, but on the client side right now we end up using all enabled repos, and since Fedora is multiarch, the behavior is going to be...not predictable. Thinking a bit about on this problem I actually happened to recall the RPM `%{_isa}` macro which is used in Fedora in various places; for example to "arch bind" `-devel` packages to their base. See for example [this case](https://src.fedoraproject.org/rpms/ostree/blob/33c7dc02bc4f14a6acd7679cc4d84e9629294ae4/f/ostree.spec#_79) in libostree. As I noted at first, the core problem here is that the "final" RPM architecture field is not symmetric in any way with the definition of that `%{_isa}` macro. See: https://github.com/rpm-software-management/rpm/blob/d9d47e01146a5d4411691a71916b1030ac7da193/installplatform#L25 The *third* solution I ended up on here is to iterate over the `Provides` on the server side and we take the first thing that matches `Provides: %{name}(whatever)`. I briefly thought about trying to somehow drive into libsolv the logic to prefer the jigdoRPM's native architecture...IIRC yum did something like that in the past but it was never done in libsolv? Anyways the dependencies here are now more correct, so other tools will also handle it. Closes: #1213 Approved by: jlebon
2018-01-18 00:25:39 +03:00
assert_file_has_content requires.txt 'glibc(.*) = '
assert_file_has_content requires.txt 'systemd(.*) = '
assert_file_has_content requires.txt 'test-pkg(.*) = 1.1-1'
# And pull it; we should download the newer version by default
do_rojig2commit
# Now we should only download 2 packages
assert_file_has_content rojig2commit-out.txt '2/[1-9][0-9]* packages to import'
for x in ${origrev} ${newrev}; do
ostree --repo=rojig-unpack-repo rev-parse ${x}
done
ostree --repo=rojig-unpack-repo fsck
ostree --repo=rojig-unpack-repo refs > rojig-refs.txt
# We should have both refs; GC will be handled by the sysroot upgrader
# via deployments, same way it is for pkg layering.
assert_file_has_content rojig-refs.txt 'rpmostree/rojig/test-pkg/1.0-1.x86__64'
assert_file_has_content rojig-refs.txt 'rpmostree/rojig/test-pkg/1.1-1.x86__64'
echo "ok rojig ♲📦 update!"
jigdo V5: Use number of objects as cache invalidation trigger Changes in a server-side tree can cause the need for clients to import different objects from packages. For example, turning on documentation. Another more subtle case is where an object might "move" from package A to B by being deleted from A - then the jigdo build process will pick the B version. We need a "cache validation key"; a way for the server to tell the client that the objects it should import from the package have changed. Initially I was thinking of using the libostree "content hash" but that would be awkward as we'd have to do an import on the server side too. After more consideration I realized a simple *count* of the number of objects actually works, because (as I note in a comment) changing a file in the tree will result in it ending up in the jigdoRPM (and count as a deletion). And obviously adding or removing objects changes the count too. In fact we could have done this *without* breaking the format by just having the client start recording the number of xattr entries, but this adds greater flexibility down the line since we can in theory change how we do cache invalidation if we *really* need to (but at the cost of triggering clients to redownload packages). Note the client logic got moved around as now we need to parse all the xattrs before we decide what packages to download. My test case here is turning on docs - I noticed this actually affects *every* package which was surprising to me; I expected at least some packages wouldn't have docs. I'll double check this. It'd be good to have a "moving object" case too which I may look at. Closes: https://github.com/projectatomic/rpm-ostree/issues/1197 Closes: #1256 Approved by: jlebon
2018-02-15 23:07:39 +03:00
# Add all docs to test https://github.com/projectatomic/rpm-ostree/issues/1197
pysetjsonmember "documentation" 'True'
runcompose --unified-core --cachedir $(pwd)/cache --add-metadata-string version=42.2
jigdo V5: Use number of objects as cache invalidation trigger Changes in a server-side tree can cause the need for clients to import different objects from packages. For example, turning on documentation. Another more subtle case is where an object might "move" from package A to B by being deleted from A - then the jigdo build process will pick the B version. We need a "cache validation key"; a way for the server to tell the client that the objects it should import from the package have changed. Initially I was thinking of using the libostree "content hash" but that would be awkward as we'd have to do an import on the server side too. After more consideration I realized a simple *count* of the number of objects actually works, because (as I note in a comment) changing a file in the tree will result in it ending up in the jigdoRPM (and count as a deletion). And obviously adding or removing objects changes the count too. In fact we could have done this *without* breaking the format by just having the client start recording the number of xattr entries, but this adds greater flexibility down the line since we can in theory change how we do cache invalidation if we *really* need to (but at the cost of triggering clients to redownload packages). Note the client logic got moved around as now we need to parse all the xattrs before we decide what packages to download. My test case here is turning on docs - I noticed this actually affects *every* package which was surprising to me; I expected at least some packages wouldn't have docs. I'll double check this. It'd be good to have a "moving object" case too which I may look at. Closes: https://github.com/projectatomic/rpm-ostree/issues/1197 Closes: #1256 Approved by: jlebon
2018-02-15 23:07:39 +03:00
newrev=$(ostree --repo=${repobuild} rev-parse ${treeref})
do_commit2rojig ${newrev}
find rojig-output -name '*.rpm' | tee rpms.txt
jigdo V5: Use number of objects as cache invalidation trigger Changes in a server-side tree can cause the need for clients to import different objects from packages. For example, turning on documentation. Another more subtle case is where an object might "move" from package A to B by being deleted from A - then the jigdo build process will pick the B version. We need a "cache validation key"; a way for the server to tell the client that the objects it should import from the package have changed. Initially I was thinking of using the libostree "content hash" but that would be awkward as we'd have to do an import on the server side too. After more consideration I realized a simple *count* of the number of objects actually works, because (as I note in a comment) changing a file in the tree will result in it ending up in the jigdoRPM (and count as a deletion). And obviously adding or removing objects changes the count too. In fact we could have done this *without* breaking the format by just having the client start recording the number of xattr entries, but this adds greater flexibility down the line since we can in theory change how we do cache invalidation if we *really* need to (but at the cost of triggering clients to redownload packages). Note the client logic got moved around as now we need to parse all the xattrs before we decide what packages to download. My test case here is turning on docs - I noticed this actually affects *every* package which was surprising to me; I expected at least some packages wouldn't have docs. I'll double check this. It'd be good to have a "moving object" case too which I may look at. Closes: https://github.com/projectatomic/rpm-ostree/issues/1197 Closes: #1256 Approved by: jlebon
2018-02-15 23:07:39 +03:00
assert_file_has_content rpms.txt 'fedora-atomic-host-42.2.*x86_64'
do_rojig2commit
jigdo V5: Use number of objects as cache invalidation trigger Changes in a server-side tree can cause the need for clients to import different objects from packages. For example, turning on documentation. Another more subtle case is where an object might "move" from package A to B by being deleted from A - then the jigdo build process will pick the B version. We need a "cache validation key"; a way for the server to tell the client that the objects it should import from the package have changed. Initially I was thinking of using the libostree "content hash" but that would be awkward as we'd have to do an import on the server side too. After more consideration I realized a simple *count* of the number of objects actually works, because (as I note in a comment) changing a file in the tree will result in it ending up in the jigdoRPM (and count as a deletion). And obviously adding or removing objects changes the count too. In fact we could have done this *without* breaking the format by just having the client start recording the number of xattr entries, but this adds greater flexibility down the line since we can in theory change how we do cache invalidation if we *really* need to (but at the cost of triggering clients to redownload packages). Note the client logic got moved around as now we need to parse all the xattrs before we decide what packages to download. My test case here is turning on docs - I noticed this actually affects *every* package which was surprising to me; I expected at least some packages wouldn't have docs. I'll double check this. It'd be good to have a "moving object" case too which I may look at. Closes: https://github.com/projectatomic/rpm-ostree/issues/1197 Closes: #1256 Approved by: jlebon
2018-02-15 23:07:39 +03:00
# Not every package has docs, but there are going to need to be changes
assert_file_has_content rojig2commit-out.txt '[1-9][0-9]*/[1-9][0-9]* packages to import ([1-9][0-9]* changed)'
ostree --repo=rojig-unpack-repo ls -R ${newrev} >/dev/null
echo "ok rojig ♲📦 updated docs"