F #6313: Add configurable backup path

This path refers to a location in the hypervisor FS, to temporally store backup
disks. The path is configured with the BACKUP_BASE_PATH variable in
datastore.conf file.

This commit also updates the references in the cancel action for the
rsync datastore

(cherry picked from commit 93ab45aec786435f0c0622638c43ed874f5a3f60)

src/datastore_mad/remotes/datastore.conf

@@ -12,3 +12,8 @@
 #   system the corresponding FS_OPTS_<FS> needs to be defines
 #   (e.g FS_OPTS_ext4="-b 16384"). If not set default MKFS options are used.
 SUPPORTED_FS="ext2,ext3,ext4,xfs"
+
+# Path to store temporary files in the hosts while backing up a VM. This path
+# needs to be the **SAME** in all hosts, although it can mount different devices
+# or FS in each one.
+#BACKUP_BASE_PATH="/path/in/the/hosts"

src/datastore_mad/remotes/rsync/backup

@@ -60,6 +60,8 @@ require_relative '../../tm/lib/tm_action'
 
 # BACKUP host:remote_dir DISK_ID:..:DISK_ID vm_uuid bj_id vm_id ds_id
 
+TransferManager::Datastore.load_env
+
 ds_xml = STDIN.read
 
 dir      = ARGV[0].split(':')
@@ -71,8 +73,12 @@ _ds_id   = ARGV[5]
 
 vm_host = dir[0]
 
-rdir   = Pathname.new(dir[1]).cleanpath.to_s
-vm_dir = Pathname.new("#{rdir}/backup/").cleanpath.to_s
+dsrdir = ENV['BACKUP_BASE_PATH']
+vm_dir = if dsrdir
+             Pathname.new("#{dsrdir}/#{vm_id}/backup").cleanpath.to_s
+         else
+             Pathname.new("#{dir[1]}/backup").cleanpath.to_s
+         end
 
 begin
     ds = REXML::Document.new(ds_xml).root
@@ -80,8 +86,6 @@ begin
     rsync_user = ds.elements['TEMPLATE/RSYNC_USER'].text
     rsync_host = ds.elements['TEMPLATE/RSYNC_HOST'].text
 
-    base = ds.elements['BASE_PATH'].text
-
     if ds.elements['TEMPLATE/RSYNC_ARGS'].nil?
         args = '-aS'
     else
@@ -181,7 +185,7 @@ script = <<~EOS
 
     BKSIZE=`du -sm #{vm_dir}`
 
-    #{ds.cmd_confinement(cmd, rdir, ['SSH_AUTH_SOCK'])} > /dev/null
+    #{ds.cmd_confinement(cmd, vm_dir, ['SSH_AUTH_SOCK'])} > /dev/null
 
     echo $BKSIZE
 EOS

src/datastore_mad/remotes/rsync/backup_cancel

@@ -55,13 +55,22 @@ require 'pathname'
 
 require 'CommandManager'
 
+require_relative '../../tm/lib/datastore'
 require_relative '../../tm/lib/tm_action'
 
+TransferManager::Datastore.load_env
+
 dir     = ARGV[0].split(':')
 vm_uuid = ARGV[1]
 
 vm_host = dir[0]
-vm_dir  = Pathname.new(dir[1]).cleanpath.to_s
+
+dsrdir = ENV['BACKUP_BASE_PATH']
+vm_dir = if dsrdir
+             Pathname.new("#{dsrdir}/#{File.basename(dir[1])}/backup").cleanpath.to_s
+         else
+             Pathname.new(dir[1]).cleanpath.to_s
+         end
 
 begin
     # Kill the pre/backup/_live scripts.

src/tm_mad/ceph/postbackup_live

@@ -54,27 +54,39 @@ $LOAD_PATH << RUBY_LIB_LOCATION
 require 'rexml/document'
 
 require_relative '../lib/tm_action'
+require_relative '../lib/datastore'
 
 #-------------------------------------------------------------------------------
 # BACKUP tm_mad host:remote_dir DISK_ID:...:DISK_ID deploy_id bjid vmid dsid
 #-------------------------------------------------------------------------------
+TransferManager::Datastore.load_env
+
 _vm_xml = STDIN.read
 
 dir       = ARGV[0].split ':'
 _disks    = ARGV[1].split ':'
 _deployid = ARGV[2]
 _bjid     = ARGV[3]
-_vmid     = ARGV[4]
+vmid      = ARGV[4]
 _dsid     = ARGV[5]
 
 rhost     = dir[0]
 rdir      = dir[1]
 
-bck_dir = "#{rdir}/backup"
+base_path = ENV['BACKUP_BASE_PATH']
+
+bck_dir = if base_path
+              "#{base_path}/#{vmid}/backup"
+          else
+              "#{rdir}/backup"
+          end
+
+rm_script = "rm -rf #{bck_dir}"
+rm_script << "; rm -rf #{base_path}/#{vmid}\n" if base_path
 
 rc = TransferManager::Action.ssh('postbackup',
                                  :host => rhost,
-                                 :cmds => "rm -rf #{bck_dir}",
+                                 :cmds => rm_script,
                                  :nostdout => false,
                                  :nostderr => false)
 

src/tm_mad/ceph/prebackup

@@ -70,6 +70,8 @@ end
 #-------------------------------------------------------------------------------
 # BACKUP tm_mad host:remote_dir DISK_ID:...:DISK_ID deploy_id bjid vmid dsid
 #-------------------------------------------------------------------------------
+TransferManager::Datastore.load_env
+
 vm_xml = STDIN.read
 
 dir       = ARGV[0].split ':'
@@ -87,7 +89,13 @@ vm      = xml_doc.root
 
 ds = TransferManager::Datastore.new(:vm_xml => vm_xml)
 
-bck_dir = "#{rdir}/backup"
+base_path = ENV['BACKUP_BASE_PATH']
+
+bck_dir = if base_path
+              "#{base_path}/#{vmid}/backup"
+          else
+              "#{rdir}/backup"
+          end
 
 expo_cmd = ''
 conv_cmd = ''

src/tm_mad/ceph/prebackup_live

@@ -73,6 +73,8 @@ end
 #-------------------------------------------------------------------------------
 # BACKUP tm_mad host:remote_dir DISK_ID:...:DISK_ID deploy_id bjid vmid dsid
 #-------------------------------------------------------------------------------
+TransferManager::Datastore.load_env
+
 vm_xml = STDIN.read
 
 dir       = ARGV[0].split ':'
@@ -90,7 +92,13 @@ vm      = xml_doc.root
 
 ds = TransferManager::Datastore.new(:vm_xml => vm_xml)
 
-bck_dir = "#{rdir}/backup"
+base_path = ENV['BACKUP_BASE_PATH']
+
+bck_dir = if base_path
+              "#{base_path}/#{vmid}/backup"
+          else
+              "#{rdir}/backup"
+          end
 
 snap_cmd = ''
 expo_cmd = ''

src/tm_mad/lib/backup_qcow2.rb

@@ -491,7 +491,7 @@ class KVMDomain
 
         @vm_dir  = opts[:vm_dir]
         @tmp_dir = "#{opts[:vm_dir]}/tmp"
-        @bck_dir = "#{opts[:vm_dir]}/backup"
+        @bck_dir = opts[:backup_dir]
 
         @socket  = "#{opts[:vm_dir]}/backup.socket"
 
@@ -1234,7 +1234,8 @@ begin
         end
     end
 
-    vm = KVMDomain.new(Base64.decode64(File.read(vxml)), :vm_dir => path)
+    vm = KVMDomain.new(Base64.decode64(File.read(vxml)), :vm_dir => path,
+                       :backup_dir => File.dirname(vxml))
 
     #---------------------------------------------------------------------------
     #  Stop operation. Only for full backups in live mode. It blockcommits

src/tm_mad/lib/datastore.rb

@@ -34,6 +34,35 @@ module TransferManager
         # file descriptor for lock
         FD = 13
 
+        # Datastore configuration files (bash syntax)
+        DATASTORECONF = '/var/lib/one/remotes/etc/datastore/datastore.conf'
+
+        @env_loaded = false
+
+        # Loads a bash formatted file to the current environment
+        # Syntax:
+        #   - Lines starting with # are ignored
+        #   - VARIABLE=VALUE
+        #   - export VARIABLE=VALUE
+        #
+        # @param [String] path to load environment from
+        def self.load_env
+            return if @env_loaded
+
+            File.readlines(DATASTORECONF).each do |l|
+                next if l.empty? || l[0] == '#'
+
+                m = l.match(/(export)?[[:blank:]]*([^=]+)="?([^"]+)"?$/)
+
+                next unless m
+
+                ENV[m[2]] = m[3].delete("\n") if m[2] && m[3]
+            end
+
+            @env_loaded = true
+        rescue StandardError
+        end
+
         # Initialize OpenNebula object and get its information
         def initialize(options = {})
             @options={

src/tm_mad/lib/kvm.rb

@@ -43,7 +43,7 @@ module TransferManager
             File.readlines(path).each do |l|
                 next if l.empty? || l[0] == '#'
 
-                m = l.match(/(export)?[[:blank:]]*([^=]+)=([^[[:blank:]]]+)$/)
+                m = l.match(/(export)?[[:blank:]]*([^=]+)=['"]?([^'"]+)['"]?$/)
 
                 next unless m
 

src/tm_mad/qcow2/postbackup

@@ -52,10 +52,12 @@ end
 $LOAD_PATH << RUBY_LIB_LOCATION
 
 require_relative '../lib/tm_action'
+require_relative '../lib/datastore'
 
 #-------------------------------------------------------------------------------
 # BACKUP tm_mad host:remote_dir DISK_ID:...:DISK_ID deploy_id bjid vmid dsid
 #-------------------------------------------------------------------------------
+TransferManager::Datastore.load_env
 
 _vm_xml = STDIN.read
 
@@ -63,17 +65,26 @@ dir       = ARGV[0].split ':'
 _disks    = ARGV[1].split ':'
 _deployid = ARGV[2]
 _bjid     = ARGV[3]
-_vmid     = ARGV[4]
+vmid      = ARGV[4]
 _dsid     = ARGV[5]
 
 rhost     = dir[0]
 rdir      = dir[1]
 
-bck_dir = "#{rdir}/backup"
+base_path = ENV['BACKUP_BASE_PATH']
+
+bck_dir = if base_path
+              "#{base_path}/#{vmid}/backup"
+          else
+              "#{rdir}/backup"
+          end
+
+rm_script = "rm -rf #{bck_dir}"
+rm_script << "; rm -rf #{base_path}/#{vmid}\n" if base_path
 
 rc = TransferManager::Action.ssh('postbackup',
                                  :host => rhost,
-                                 :cmds => "rm -rf #{bck_dir}",
+                                 :cmds => rm_script,
                                  :nostdout => false,
                                  :nostderr => false)
 

src/tm_mad/qcow2/postbackup_live

@@ -52,10 +52,12 @@ end
 $LOAD_PATH << RUBY_LIB_LOCATION
 
 require_relative '../lib/tm_action'
+require_relative '../lib/datastore'
 
 #-------------------------------------------------------------------------------
 # BACKUP tm_mad host:remote_dir DISK_ID:...:DISK_ID deploy_id bjid vmid dsid
 #-------------------------------------------------------------------------------
+TransferManager::Datastore.load_env
 
 _vm_xml = STDIN.read
 
@@ -63,14 +65,20 @@ dir        = ARGV[0].split ':'
 disks      = ARGV[1]
 _deploy_id = ARGV[2]
 _bjid      = ARGV[3]
-_vmid      = ARGV[4]
+vmid       = ARGV[4]
 _dsid      = ARGV[5]
 
 rhost     = dir[0]
 rdir      = dir[1]
 
+base_path = ENV['BACKUP_BASE_PATH']
+
 tmp_dir = "#{rdir}/tmp"
-bck_dir = "#{rdir}/backup"
+bck_dir = if base_path
+              "#{base_path}/#{vmid}/backup"
+          else
+              "#{rdir}/backup"
+          end
 
 qcow2_util = '/var/tmp/one/tm/lib/backup_qcow2.rb'
 
@@ -89,6 +97,8 @@ post_script = <<~EOS
     rm -rf #{bck_dir}
 EOS
 
+post_script << "    rm -rf #{base_path}/#{vmid}\n" if base_path
+
 rc = TransferManager::Action.ssh('postbackup_live',
                                  :host => rhost,
                                  :cmds => post_script,

src/tm_mad/qcow2/prebackup

@@ -59,19 +59,28 @@ require 'rexml/document'
 #-------------------------------------------------------------------------------
 # BACKUP tm_mad host:remote_dir DISK_ID:...:DISK_ID deploy_id vmid bjid dsid
 #-------------------------------------------------------------------------------
+TransferManager::Datastore.load_env
+
 vm_xml = STDIN.read
 
 dir       = ARGV[0].split ':'
 disks     = ARGV[1]
 _deployid = ARGV[2]
 _bjid     = ARGV[3]
-_vmid     = ARGV[4]
+vmid      = ARGV[4]
 _dsid     = ARGV[5]
 
 rhost     = dir[0]
 rdir      = dir[1]
 
-bck_dir    = "#{rdir}/backup"
+base_path = ENV['BACKUP_BASE_PATH']
+
+bck_dir = if base_path
+              "#{base_path}/#{vmid}/backup"
+          else
+              "#{rdir}/backup"
+          end
+
 qcow2_util = '/var/tmp/one/tm/lib/backup_qcow2.rb'
 qcow2_cmd  = "#{qcow2_util} -d \"#{disks}\" -x #{bck_dir}/vm.xml -p #{rdir}"
 

src/tm_mad/qcow2/prebackup_live

@@ -54,29 +54,33 @@ $LOAD_PATH << RUBY_LIB_LOCATION
 require 'rexml/document'
 
 require_relative '../lib/tm_action'
-require_relative '../lib/kvm'
 require_relative '../lib/datastore'
 
-# TODO: fsfreeze for each hypervisor based on VM_MAD
-include TransferManager::KVM
-
 #-------------------------------------------------------------------------------
 # BACKUP tm_mad host:remote_dir DISK_ID:...:DISK_ID deploy_id vmid bjid dsid
 #-------------------------------------------------------------------------------
+TransferManager::Datastore.load_env
+
 vm_xml = STDIN.read
 
 dir        = ARGV[0].split ':'
 disks      = ARGV[1]
 _deploy_id = ARGV[2]
 _bjid      = ARGV[3]
-_vmid      = ARGV[4]
+vmid       = ARGV[4]
 _dsid      = ARGV[5]
 
 rhost     = dir[0]
 rdir      = dir[1]
 
+base_path = ENV['BACKUP_BASE_PATH']
+
 tmp_dir = "#{rdir}/tmp"
-bck_dir = "#{rdir}/backup"
+bck_dir = if base_path
+              "#{base_path}/#{vmid}/backup"
+          else
+              "#{rdir}/backup"
+          end
 
 qcow2_util = '/var/tmp/one/tm/lib/backup_qcow2.rb'
 qcow2_cmd  = "#{qcow2_util} -l -d \"#{disks}\" -x #{bck_dir}/vm.xml -p #{rdir}"