tests: Fix installed tests more

OK so I noticed that something was failing and we were missing
`set -xeuo pipefail` in our shells.  That of course revealed
the ansible tests didn't actually work - my only defense
here is spending so much time fighting to get it through CI
and trying something new.

Anyways, to make the staged-deploy tests work we need a task
that actually uses `rpm-ostree override` rather than `usroverlay`.

Let's make this a bit saner and have a clean split between
tests that are "shell-script+usroverlay" and "ansible+override".

Closes: #1577
Approved by: jlebon

tests/installed/destructive-ansible.yml

@@ -0,0 +1,17 @@
+# Ansible-based tests.
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  vars:
+    use_git_build: True
+    tests: "."
+  tasks:
+    - import_tasks: tasks/query-host.yml
+    - set_fact:
+        rpmostree_initial_deployment: "{{ rpmostree_status[\"deployments\"][0] }}"
+    - import_tasks: tasks/install-git.yml
+      when: use_git_build
+
+    - import_tasks: destructive/staged-deploy.yml

tests/installed/destructive-unit.yml

@@ -1,4 +1,7 @@
-# This entrypoint right now just runs the sysinstalled-tests.
+# This entrypoint right now just runs shell-script based tests
+# from destructive/.  Note that we `rpm-ostree usroverlay` git
+# builds.  So it's not supported to reboot in these tests.
+# These tests will be run serially, and can e.g. change deployments.
 ---
 - hosts: localhost
   tags:
@@ -17,9 +20,6 @@
     - name: Copy test data
       synchronize: src=../../ dest=/root/tests/ archive=yes
 
-    # First, the Ansible-based tests
-    - import_tasks: destructive/staged-deploy.yml
-
     - find:
         paths: /root/tests/installed/destructive
         patterns: "itest-*.sh"

tests/installed/destructive/staged-deploy.yml

@@ -3,6 +3,7 @@
 
 - name: Write staged-deploy commit
   shell: |
+    set -xeuo pipefail
     cd /ostree/repo/tmp
     # https://github.com/ostreedev/ostree/issues/1569
     ostree checkout -H ${commit} t
@@ -11,7 +12,7 @@
     orig_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy)
     ostree admin deploy --stage staged-deploy
     new_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy)
-    assert_not_streq "${orig_mtime}" "${new_mtime}"
+    test "${orig_mtime}" != "${new_mtime}"
     test -f /run/ostree/staged-deployment
     ostree refs | grep -E -e '^ostree/' | while read ref; do
       if test "$(ostree rev-parse ${ref})" = "${newcommit}"; then
@@ -25,6 +26,7 @@
 - include_tasks: ../tasks/reboot.yml
 - name: Check that deploy-staged service worked
   shell: |
+    set -xeuo pipefail
     # Assert that the previous boot had a journal entry for it
     journalctl -b "-1" -u ostree-finalize-staged.service | grep -q -e 'Transaction complete'
     # And there should not be a staged deployment
@@ -33,11 +35,13 @@
   shell: rpm-ostree rollback
 - include_tasks: ../tasks/reboot.yml
 - shell: |
+    set -xeuo pipefail
     rpm-ostree cleanup -rp
 
 # Ensure we can unstage
 - name: Write staged-deploy commit, then unstage
   shell: |
+    set -xeuo pipefail
     ostree admin deploy --stage staged-deploy
     ostree admin status > status.txt
     grep -qFe '(staged)' status.txt
@@ -49,9 +53,9 @@
   environment:
     commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}"
 
-# Staged should be overwritten by non-staged
-- name: Write staged-deploy commit, then unstage
+- name: Staged should be overwritten by non-staged
   shell: |
+    set -xeuo pipefail
     ostree admin deploy --stage staged-deploy
     test -f /run/ostree/staged-deployment
     ostree --repo=/ostree/repo refs --create nonstaged-deploy "${commit}"

tests/installed/run.sh

@@ -6,7 +6,7 @@ set -xeuo pipefail
 dn=$(cd $(dirname $0) && pwd)
 
 # TODO: parallelize this
-PLAYBOOKS=${PLAYBOOKS:-nondestructive.yml destructive.yml}
+PLAYBOOKS=${PLAYBOOKS:-nondestructive.yml destructive-ansible.yml destructive-unit.yml}
 for playbook in $PLAYBOOKS; do
     time ${dn}/playbook-run.sh -v ${dn}/${playbook}
 done

tests/installed/tasks/install-git.yml

@@ -0,0 +1,19 @@
+# Use package layering to install git builds.
+- command: ostree --version
+  changed_when: False
+  register: ostree_orig_version
+- set_fact:
+    ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}"
+- name: Copy locally built RPMs
+  synchronize: src=build/x86_64/ dest=/root/x86_64/ archive=yes
+- name: Install RPMs
+  shell: rpm-ostree override replace /root/x86_64/*.rpm
+- include_tasks: ../tasks/reboot.yml
+- command: ostree --version
+  register: ostree_new_version
+- set_fact:
+    ostree_new_version_yaml: "{{ ostree_new_version.stdout | from_yaml }}"
+- name: "Fail if we didn't change the ostree version"
+  when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git']
+  fail:
+    msg: "Failed to change ostree version"