Redo vmcheck to abstract over ssh-config, drop all building

My development environment is now using "pet" docker containers.
I use VMs for testing things that require that (like rpm-ostree).

This patch builds on work from @jlebon in
https://github.com/projectatomic/rpm-ostree/pull/509
to rework `vmcheck` such that it can work on any `ssh-config`.  By
default we expect this to be Vagrant.

However, I go a lot farther and delete the `vmbuild` code that was
trying to do builds in a container on the target VM.  I think this is
still worth pursuing at some point, but for now I think it's
reasonable to assume that the rpm-ostree developer audience uses Linux
as their host workstation and hence has containers.

(There's another important point here in that for developing lower
 level things like rpm-ostree, there's a strong push to make the VM
 disposable and not a pet)

Closes: #516
Approved by: jlebon
This commit is contained in:
Colin Walters 2016-11-14 12:07:43 -05:00 committed by Atomic Bot
parent 1577afd85c
commit 2f9d586bdc
23 changed files with 168 additions and 246 deletions

View File

@ -91,24 +91,19 @@ check-local:
@echo " *** NOTE ***"
@echo " *** NOTE ***"
.PHONY: vmbuild vmshell vmcheck testenv
.PHONY: vmsync vmshell vmcheck testenv
vmbuild:
@if [ -z "$(SKIP_VMBUILD)" ]; then \
env $(BASE_TESTS_ENVIRONMENT) \
tests/utils/vmbuild.sh; \
fi
vmsync:
@env $(BASE_TESTS_ENVIRONMENT) ./tests/vmcheck/sync.sh
vmoverlay:
@env $(BASE_TESTS_ENVIRONMENT) \
tests/utils/vmoverlay.sh
@env $(BASE_TESTS_ENVIRONMENT) ./tests/vmcheck/overlay.sh
vmshell: vmbuild
sleep 2
vmshell: vmsync
ssh -F ssh-config vmcheck
# set up test environment to somewhat resemble uninstalled tests
vmcheck: vmbuild tests/common/compose/yum/repo/repodata/repomd.xml
vmcheck: vmoverlay tests/common/compose/yum/repo/repodata/repomd.xml
@env VMTESTS=1 $(BASE_TESTS_ENVIRONMENT) \
sh tests/vmcheck/test.sh

View File

@ -42,7 +42,7 @@ AM_LDFLAGS += "-Wl,-rpath=$(libdir)/rpm-ostree"
EXTRA_DIST += autogen.sh COPYING
GITIGNOREFILES += build-aux/ gtk-doc.make config.h.in aclocal.m4
GITIGNOREFILES += build-aux/ gtk-doc.make config.h.in aclocal.m4 insttree/
SED_SUBST = sed \
-e 's,[@]libexecdir[@],$(libexecdir),g' \

View File

@ -1,24 +0,0 @@
# CentOS-Base.repo
#
# The mirror system uses the connecting IP address of the client and the
# update status of each mirror to pick mirrors that are updated to and
# geographically close to the client. You should use this for CentOS updates
# unless you are manually picking other mirrors.
#
# If the mirrorlist= does not work for you, as a fall back you can try the
# remarked out baseurl= line instead.
#
#
[base]
name=CentOS-$releasever - Base
mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/
gpgcheck=0
#released updates
[updates]
name=CentOS-$releasever - Updates
mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/updates/$basearch/
gpgcheck=0

View File

@ -1,9 +0,0 @@
FROM centos
ADD centos-atomic-7.repo /etc/yum.repos.d/centos-atomic-7.repo
ADD local.repo /etc/yum.repos.d/local.repo
ADD repo /var/tmp/repo
ADD pkgblob.tar /var/tmp/centos
ADD test-centos.json /var/tmp/test/test-centos.json
ADD test-centos.repo /var/tmp/test/test-centos.repo
ADD container-build.sh /usr/bin/container-build
RUN /usr/bin/container-build

View File

@ -1,5 +0,0 @@
[centos-atomic7-testing]
name=centos-atomic7-testing
baseurl=https://cbs.centos.org/repos/atomic7-testing/x86_64/os/
enabled=1
gpgcheck=0

View File

@ -1,9 +0,0 @@
#!/bin/sh
set -xeuo pipefail
yum -y install rpm-ostree
cd /var/tmp
mkdir -p test
cd test
mkdir repo
ostree --repo=repo init --mode=bare-user

View File

@ -1,4 +0,0 @@
[local]
name=local
baseurl=file:///var/tmp/repo
gpgcheck=0

View File

@ -1,6 +0,0 @@
{
"ref": "testos/x86_64/test",
"repos": ["test-centos"],
"packages": ["kernel", "systemd"],
"selinux": true
}

View File

@ -1,4 +0,0 @@
[test-centos]
name=test-centos
baseurl=file:///var/tmp/centos/pkgblob
gpgcheck=0

View File

@ -28,17 +28,26 @@ vm_setup() {
fi
local sshopts="-F ${topsrcdir}/ssh-config \
-o User=root \
-o ControlMaster=auto \
-o ControlPath=${topsrcdir}/ssh.sock \
-o ControlPersist=yes"
export SSH="ssh $sshopts vmcheck"
export SCP="scp $sshopts"
if grep -q 'User.*vagrant' ${topsrcdir}/ssh-config; then
export using_sshfs=yes
else
export using_sshfs=no
fi
}
vm_rsync() {
if test ${using_sshfs} = yes; then
return
fi
pushd ${topsrcdir}
rsync -az --no-owner --no-group --filter ":- .gitignore" \
-e "ssh -F ssh-config" --exclude .git/ . vmcheck:/root/sync
-e "ssh -F ssh-config" --exclude .git/ . vmcheck:/root/sync
popd
}
@ -82,7 +91,8 @@ vm_ssh_wait() {
sleep 1
done
# final check at the timeout mark
vm_cmd true &> /dev/null
set -x
vm_cmd true
}
# reboot the vm

View File

@ -1,15 +0,0 @@
#!/bin/bash
set -euo pipefail
source ${commondir}/libvm.sh
# create ssh-config if needed and export cmds
vm_setup
if [ -n "${VMCLEAN:-}" ]; then
vm_cmd rm -rf sync
fi
vm_rsync
vm_cmd make -C sync/vagrant install VERSION=$(git describe)
vm_reboot

View File

@ -1,16 +0,0 @@
#!/bin/bash
set -euo pipefail
source ${commondir}/libvm.sh
# create ssh-config if needed and export cmds
vm_setup
if [ -n "${VMCLEAN:-}" ]; then
vm_cmd rm -rf sync
fi
vm_rsync
vm_cmd make -C sync/vagrant ofsinstall
vm_cmd systemctl daemon-reload
vm_cmd systemctl restart rpm-ostreed

44
tests/vmcheck/overlay.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
set -euo pipefail
# Execute this code path on the host
if test -z "${OVERLAY_IN_VM:-}"; then
. ${commondir}/libvm.sh
set -x
topdir=$(git rev-parse --show-toplevel)
cd ${topdir}
rm insttree -rf
make install DESTDIR=$(pwd)/insttree
vm_rsync
ssh -o User=root -F ssh-config vmcheck "env OVERLAY_IN_VM=1 ~vagrant/sync/tests/vmcheck/overlay.sh"
exit 0
fi
set -x
# And then this code path in the VM
commit=$(rpm-ostree status --json | \
python -c '
import sys, json;
deployments = json.load(sys.stdin)["deployments"]
for deployment in deployments:
if deployment["booted"]:
print deployment["checksum"]
exit()')
if [[ -z $commit ]] || ! ostree rev-parse $commit; then
echo "Error while determining current commit" >&2
exit 1
fi
cd /ostree/repo/tmp
rm vmcheck -rf
ostree checkout $commit vmcheck --fsync=0
rsync -rv ~vagrant/sync/insttree/usr/ vmcheck/usr/
ostree refs --delete vmcheck || true
ostree commit -b vmcheck -s '' --tree=dir=vmcheck --link-checkout-speedup
ostree admin deploy vmcheck
systemctl reboot

10
tests/vmcheck/sync.sh Normal file
View File

@ -0,0 +1,10 @@
#!/bin/bash
set -euo pipefail
. ${commondir}/libvm.sh
set -x
cd ${builddir}
rm insttree -rf
make install DESTDIR=$(pwd)/insttree
ssh -o User=root -F ssh-config vmcheck "ostree admin unlock; cd ~vagrant/sync && rsync -rv insttree/usr/ /usr/ && systemctl restart rpm-ostreed"

View File

@ -12,6 +12,8 @@ if ! vm_ssh_wait 20; then
exit 1
fi
echo "VM is running."
# just error out if we're unlocked -- we use the current deployment as the
# fallback between each test, so we need to be sure it's in a state that works.
# also, the user might have forgotten that these tests are somewhat destructive

0
vagrant/.gitignore vendored Normal file
View File

View File

@ -1,10 +0,0 @@
FROM centos/tools
ADD atomic-centos-continuous.repo /etc/yum.repos.d/atomic-centos-continuous.repo
RUN yum -y install yum-plugin-priorities sudo && \
yum -y install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ \
grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux \
which xz python gcc gperf 'pkgconfig(libsystemd)' cmake expat-devel python-devel check-devel \
python-sphinx createrepo_c \
&& yum-builddep -y rpm-ostree
LABEL RUN "/usr/bin/docker run --privileged -ti -v /var/roothome:/root -v /etc:/host/etc -v /usr:/host/usr \${IMAGE}"
WORKDIR /root/sync

View File

@ -1,56 +0,0 @@
all:
echo "Targets: buildimg build install ofsinstall check clean"
BUILDER_ARGS = /usr/bin/docker run --rm --privileged -v /var/roothome:/root
BUILDER_IMG = rpm-ostree-builder
BUILDER_RUN = $(BUILDER_ARGS) $(BUILDER_IMG)
.PHONY: dockerwait buildimg build install ofsinstall check clean
# necessary precaution in case we're operating on a freshly
# booted VM
dockerwait:
@echo "Waiting for docker service to start..."; \
timeout=10; \
while [ $$timeout -gt 0 ]; do \
if systemctl show -p SubState docker.service | grep -q =running; then \
exit 0; \
fi; \
timeout=$$((timeout - 1)); \
sleep 1; \
done; \
echo "Waited too long for docker service to start"; \
exit 1
buildimg: dockerwait
docker build -t $(BUILDER_IMG) -f Dockerfile.builder .
build: buildimg
if ! test -f ../configure; then \
$(BUILDER_RUN) env NOCONFIGURE=1 ./autogen.sh; \
fi
if ! test -f ../Makefile; then \
$(BUILDER_RUN) env CFLAGS='$(CFLAGS)' \
./configure --prefix=/usr --libdir=/usr/lib64; \
fi
$(BUILDER_RUN) make -j4 CFLAGS="$(CFLAGS)"
install: build
sh checkout.sh
$(BUILDER_ARGS) -v /etc:/host/etc \
-v /ostree/repo/tmp/vmcheck.ro/usr:/host/usr \
$(BUILDER_IMG) make install DESTDIR=/host
VERSION=$(VERSION) sh commit_and_deploy.sh
ofsinstall: build
ostree admin unlock || :
$(BUILDER_ARGS) -v /etc:/host/etc \
-v /usr:/host/usr \
$(BUILDER_IMG) make install DESTDIR=/host
check: build
$(BUILDER_RUN) make check
clean: buildimg
$(BUILDER_RUN) make clean

View File

@ -1 +1,38 @@
These are helper files for setting up the vagrant machine.
Developing using Docker + Vagrant
---------------------------------
The current tooling here is oriented towards
doing builds inside a CentOS 7 pet container,
with Vagrant on the host.
You should share the git working directory with the c7 container.
Assuming you have git repositories stored in `/srv`, something like:
```
docker run --name c7dev --privileged -v /srv:/srv --net=host -ti centos bash
```
You can start the Vagrant box. To work around "fuse-sshfs" not
being built into the Vagrant box, do something like this:
```
vagrant up ; vagrant provision; vagrant halt; vagrant up
```
Now, once you do a build inside the c7dev container, like:
```
./autogen.sh CFLAGS='-ggdb -O0' --prefix=/usr --libdir=/usr/lib64 --enable-installed-tests --enable-gtk-doc
```
To sync over and install the built binaries:
```
make vmsync
```
You may also want to use `vmcheck`, like this:
```
make vmoverlay && make vmcheck
```

View File

@ -1,4 +0,0 @@
[atomic-centos-continuous]
baseurl=https://ci.centos.org/artifacts/sig-atomic/rdgo/centos-continuous/build
gpgcheck=0
priority=1

View File

@ -1,24 +0,0 @@
#!/bin/bash
set -euo pipefail
# ugly but simple way of fetching commit we're sitting on
commit=$(rpm-ostree status --json | \
python -c '
import sys, json;
deployments = json.load(sys.stdin)["deployments"]
for deployment in deployments:
if deployment["booted"]:
print deployment["checksum"]
exit()')
if [[ -z $commit ]] || ! ostree rev-parse $commit; then
echo "Error while determining current commit" >&2
exit 1
fi
cd /ostree/repo/tmp
umount vmcheck.ro 2>/dev/null ||:
rm -rf vmcheck*
ostree checkout $commit vmcheck --fsync=0
mkdir vmcheck.ro
rofiles-fuse vmcheck{,.ro}

View File

@ -1,17 +0,0 @@
#!/bin/bash
set -euo pipefail
cd /ostree/repo/tmp
umount vmcheck.ro
cmd="ostree commit -b vmcheck -s '' \
--tree=dir=vmcheck \
--link-checkout-speedup"
if [ -n "${VERSION:-}" ]; then
cmd="$cmd --add-metadata-string=version=$VERSION"
fi
eval $cmd
ostree admin deploy vmcheck
sync

View File

@ -3,37 +3,64 @@
gather_facts: no
become: yes
tasks:
- name: generate config
local_action: shell vagrant ssh-config > ssh-config
become: no
# if we're not already using the CAHC box, then add the
# remote to make it easier for the user to rebase later
- name: add CAHC ostree remote
command: >
ostree remote add --set=gpg-verify=false centos-atomic-continuous
https://ci.centos.org/artifacts/sig-atomic/rdgo/centos-continuous/ostree/repo/
args:
creates: /etc/ostree/remotes.d/centos-atomic-continuous.conf
# We generate a valid ssh-config here that libvm.sh can
# make use of. This also requires making sure the root
# user can be ssh'ed in directly.
# set up auth key
# The test suite requires direct ssh to root.
- file: state=directory mode=0600 path=/root/.ssh
- command: cp .ssh/authorized_keys /root/.ssh
- copy: remote_src=True src=/home/vagrant/.ssh/authorized_keys dest=/root/.ssh/authorized_keys
# make sure root account is unlocked
- name: unlock root account
command: passwd -u root
shell:
cmd: 'passwd -u root && touch /root/.unlocked'
creates: /root/.unlocked
# generate ssh config
- name: generate config
local_action: shell vagrant ssh-config > ../ssh-config
become: no
- copy:
dest: /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
content: |
-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: GnuPG v1.4.11 (GNU/Linux)
- name: make sure user in config is root
local_action: lineinfile
dest=../ssh-config
regexp='^( *User) .*$'
line='\1 root'
backrefs=yes
become: no
mQINBFKuaIQBEAC1UphXwMqCAarPUH/ZsOFslabeTVO2pDk5YnO96f+rgZB7xArB
OSeQk7B90iqSJ85/c72OAn4OXYvT63gfCeXpJs5M7emXkPsNQWWSju99lW+AqSNm
jYWhmRlLRGl0OO7gIwj776dIXvcMNFlzSPj00N2xAqjMbjlnV2n2abAE5gq6VpqP
vFXVyfrVa/ualogDVmf6h2t4Rdpifq8qTHsHFU3xpCz+T6/dGWKGQ42ZQfTaLnDM
jToAsmY0AyevkIbX6iZVtzGvanYpPcWW4X0RDPcpqfFNZk643xI4lsZ+Y2Er9Yu5
S/8x0ly+tmmIokaE0wwbdUu740YTZjCesroYWiRg5zuQ2xfKxJoV5E+Eh+tYwGDJ
n6HfWhRgnudRRwvuJ45ztYVtKulKw8QQpd2STWrcQQDJaRWmnMooX/PATTjCBExB
9dkz38Druvk7IkHMtsIqlkAOQMdsX1d3Tov6BE2XDjIG0zFxLduJGbVwc/6rIc95
T055j36Ez0HrjxdpTGOOHxRqMK5m9flFbaxxtDnS7w77WqzW7HjFrD0VeTx2vnjj
GqchHEQpfDpFOzb8LTFhgYidyRNUflQY35WLOzLNV+pV3eQ3Jg11UFwelSNLqfQf
uFRGc+zcwkNjHh5yPvm9odR1BIfqJ6sKGPGbtPNXo7ERMRypWyRz0zi0twARAQAB
tChGZWRvcmEgRVBFTCAoNykgPGVwZWxAZmVkb3JhcHJvamVjdC5vcmc+iQI4BBMB
AgAiBQJSrmiEAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBqL66iNSxk
5cfGD/4spqpsTjtDM7qpytKLHKruZtvuWiqt5RfvT9ww9GUUFMZ4ZZGX4nUXg49q
ixDLayWR8ddG/s5kyOi3C0uX/6inzaYyRg+Bh70brqKUK14F1BrrPi29eaKfG+Gu
MFtXdBG2a7OtPmw3yuKmq9Epv6B0mP6E5KSdvSRSqJWtGcA6wRS/wDzXJENHp5re
9Ism3CYydpy0GLRA5wo4fPB5uLdUhLEUDvh2KK//fMjja3o0L+SNz8N0aDZyn5Ax
CU9RB3EHcTecFgoy5umRj99BZrebR1NO+4gBrivIfdvD4fJNfNBHXwhSH9ACGCNv
HnXVjHQF9iHWApKkRIeh8Fr2n5dtfJEF7SEX8GbX7FbsWo29kXMrVgNqHNyDnfAB
VoPubgQdtJZJkVZAkaHrMu8AytwT62Q4eNqmJI1aWbZQNI5jWYqc6RKuCK6/F99q
thFT9gJO17+yRuL6Uv2/vgzVR1RGdwVLKwlUjGPAjYflpCQwWMAASxiv9uPyYPHc
ErSrbRG0wjIfAR3vus1OSOx3xZHZpXFfmQTsDP7zVROLzV98R3JwFAxJ4/xqeON4
vCPFU6OsT3lWQ8w7il5ohY95wmujfr6lk89kEzJdOTzcn7DBbUru33CQMGKZ3Evt
RjsC7FDbL017qxS+ZVA/HGkyfiu4cpgV8VUnbql5eAZ+1Ll6Dw==
=hdPa
-----END PGP PUBLIC KEY BLOCK-----
- name: Add repository
yum_repository:
name: epel
description: EPEL YUM repo
mirrorlist: 'https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch'
gpgcheck: yes
gpgkey: file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
- command: rpm -q fuse-sshfs
register: rpmq_fuse_sshfs
changed_when: False
failed_when: False
- command: rpm-ostree install fuse-sshfs
when: rpmq_fuse_sshfs.rc != 0