ovirt-imageio/test/backup.py
Albert Esteve fc6e58d9e6 reuse: addheader test/*.py
Add SPDX header to python files with
the 'py' extension in the test directory.

Signed-off-by: Albert Esteve <aesteve@redhat.com>
2022-10-18 13:04:20 +02:00

265 lines
8.0 KiB
Python

# SPDX-FileCopyrightText: Red Hat, Inc.
# SPDX-License-Identifier: GPL-2.0-or-later
import logging
import time
from contextlib import contextmanager
from urllib.parse import urlparse
from ovirt_imageio._internal import nbd
from ovirt_imageio._internal import nbdutil
from ovirt_imageio._internal import qemu_img
from ovirt_imageio._internal import qemu_nbd
from . import qemu
from . import qmp
log = logging.getLogger("backup")
@contextmanager
def full_backup(tmpdir, disk, fmt, sock, checkpoint=None):
"""
Start qemu internal nbd server using address sock, exposing disk for
full backup, creating temporary files in tmpdir.
"""
scratch_disk = str(tmpdir.join("scratch.qcow2"))
qmp_sock = nbd.UnixAddress(tmpdir.join("qmp.sock"))
disk_size = qemu_img.info(disk)["virtual-size"]
qemu_img.create(scratch_disk, "qcow2", size=disk_size)
with qemu.run(disk, fmt, qmp_sock, start_cpu=False, shutdown_timeout=10), \
qmp.Client(qmp_sock) as c, \
run(c, sock, scratch_disk, checkpoint=checkpoint):
yield
@contextmanager
def run(qmp, sock, scratch_disk, checkpoint=None, incremental=None):
b = Backup(qmp, sock, scratch_disk, checkpoint=checkpoint,
incremental=incremental)
b.start()
try:
yield b
finally:
b.stop()
class Backup:
def __init__(self, qmp, sock, scratch_disk, checkpoint=None,
incremental=None):
self.qmp = qmp
self.sock = sock
self.scratch_disk = scratch_disk
self.checkpoint = checkpoint
self.incremental = incremental
# Hardcoded value, good enough for now.
self.export = "sda"
# Use node name "file0" as a stable reference to our disk. It will not
# change when the block graph is modifed during backup.
self.file = "file0"
self.job = "job0"
# Libvirt uses something like "backup-libvirt-42-format".
self.node = "backup-file0"
self.bitmap = self.node if self.incremental else None
def start(self):
log.info("Starting backup checkpoint=%s incremental=%s",
self.checkpoint, self.incremental)
self.start_nbd_server()
self.add_backup_node()
if self.incremental:
self.add_dirty_bitmap()
self.run_backup_transaction()
self.add_to_nbd_server()
def stop(self):
log.info("Stopping backup checkpoint=%s incremental=%s",
self.checkpoint, self.incremental)
# Give qemu time to detect that the NBD client disconnected before
# we tear down the nbd server.
log.debug("Waiting before tearing down nbd server")
time.sleep(0.1)
self.remove_from_nbd_server()
self.stop_nbd_server()
self.cancel_block_job()
self.remove_backup_node()
if self.incremental:
self.remove_dirty_bitmap()
def start_nbd_server(self):
log.debug("Starting nbd server listening on %s", self.sock)
if self.sock.transport == "unix":
addr = {"type": "unix",
"data": {"path": self.sock.path}}
elif self.sock.transport == "tcp":
addr = {"type": "inet",
"data": {"host": self.sock.host,
"port": str(self.sock.port)}}
else:
raise RuntimeError("Unsupported transport: {}".format(self.sock))
self.qmp.execute("nbd-server-start", {"addr": addr})
def add_backup_node(self):
log.debug("Adding backup node %s for %s", self.node, self.file)
self.qmp.execute("blockdev-add", {
"driver": "qcow2",
"node-name": self.node,
"file": {
"driver": "file",
"filename": self.scratch_disk,
},
"backing": self.file,
})
def add_dirty_bitmap(self):
"""
Real backup code get a list of all checkpoints since incremental, and
merge all of them into the temporary bitmap.
"""
log.debug("Adding dirty bitmap %s for incremental backup since: %s",
self.bitmap, self.incremental)
self.qmp.execute("block-dirty-bitmap-add", {
"node": self.file,
"name": self.bitmap,
"disabled": True,
})
self.qmp.execute("block-dirty-bitmap-merge", {
"node": self.file,
"target": self.bitmap,
"bitmaps": [self.incremental],
})
def run_backup_transaction(self):
log.debug("Running backup transaction")
actions = []
if self.incremental:
# Disbale the previous active dirty bitmap. Changes after this
# point will be recorded in the new bitmap.
actions.append({
"type": "block-dirty-bitmap-disable",
"data": {
"node": self.file,
"name": self.incremental,
}
})
if self.checkpoint:
actions.append({
"type": "block-dirty-bitmap-add",
"data": {
"name": self.checkpoint,
"node": self.file,
"persistent": True,
}
})
actions.append({
"type": "blockdev-backup",
"data": {
"device": self.file,
"job-id": self.job,
"sync": "none",
"target": self.node,
}
})
self.qmp.execute("transaction", {"actions": actions})
def add_to_nbd_server(self):
log.debug("Adding node %s and bitmap %s to nbd server",
self.node, self.bitmap)
arguments = {
"type": "nbd",
"id": self.export,
"node-name": self.node,
"name": self.export,
"allocation-depth": True,
}
if self.bitmap:
arguments["bitmaps"] = [self.bitmap]
self.qmp.execute("block-export-add", arguments)
def remove_from_nbd_server(self):
log.debug("Removing export %s from nbd server", self.export)
self.qmp.execute("block-export-del", {
"id": self.export,
"mode": "hard",
})
def stop_nbd_server(self):
log.debug("Stopping nbd server")
self.qmp.execute("nbd-server-stop")
def cancel_block_job(self):
log.debug("Cancelling block job %s", self.job)
self.qmp.execute("block-job-cancel", {"device": self.job})
def remove_backup_node(self):
log.debug("Removing backup node %s", self.node)
self.qmp.execute("blockdev-del", {"node-name": self.node})
def remove_dirty_bitmap(self):
log.debug("Removing dirty bitmap %s from node %s",
self.bitmap, self.file)
self.qmp.execute("block-dirty-bitmap-remove", {
"node": self.file,
"name": self.bitmap,
})
# Backup data helpers.
def copy_disk(nbd_url, backup_disk):
log.info("Backing up data extents from %s to %s", nbd_url, backup_disk)
backup_url = urlparse(nbd_url)
with nbd.open(backup_url) as src_client, \
qemu_nbd.open(backup_disk, "qcow2") as dst_client:
nbdutil.copy(src_client, dst_client)
def copy_dirty(nbd_url, backup_disk):
log.info("Backing up dirty extents from %s to %s", nbd_url, backup_disk)
backup_url = urlparse(nbd_url)
with nbd.open(backup_url, dirty=True) as src_client, \
qemu_nbd.open(backup_disk, "qcow2") as dst_client:
buf = bytearray(4 * 1024**2)
offset = 0
for ext in nbdutil.extents(src_client, dirty=True):
if ext.dirty:
todo = ext.length
while todo:
step = min(todo, len(buf))
view = memoryview(buf)[:step]
src_client.readinto(offset, view)
dst_client.write(offset, view)
offset += step
todo -= step
else:
offset += ext.length