diff --git a/.gitignore b/.gitignore index 739c281c6a..ffc6610034 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,10 @@ share/esx-fw-vnc/.vagrant* share/context/* !share/context/download_context.sh !share/context/SConstruct + +# editors temporary/swap files +*.swp +*.swo + +# backups +*.*- diff --git a/install.sh b/install.sh index 15c20aac6f..5bfcd1a43f 100755 --- a/install.sh +++ b/install.sh @@ -262,7 +262,6 @@ SHARE_DIRS="$SHARE_LOCATION/examples \ $SHARE_LOCATION/conf \ $SHARE_LOCATION/context \ $SHARE_LOCATION/onecfg - $SHARE_LOCATION/onecfg/augeas \ $SHARE_LOCATION/onecfg/etc" ETC_DIRS="$ETC_LOCATION/vmm_exec \ @@ -314,7 +313,8 @@ LIB_DIRS="$LIB_LOCATION/ruby \ $LIB_LOCATION/onecfg/lib/config \ $LIB_LOCATION/onecfg/lib/config/type \ $LIB_LOCATION/onecfg/lib/config/type/augeas \ - $LIB_LOCATION/onecfg/lib/config/type/yaml" + $LIB_LOCATION/onecfg/lib/config/type/yaml \ + $LIB_LOCATION/onecfg/lib/patch" VAR_DIRS="$VAR_LOCATION/remotes \ $VAR_LOCATION/remotes/etc \ @@ -760,7 +760,7 @@ INSTALL_ONECFG_FILES=( ONECFG_LIB_CONFIG_TYPE_FILES:$LIB_LOCATION/onecfg/lib/config/type ONECFG_LIB_CONFIG_TYPE_AUGEAS_FILES:$LIB_LOCATION/onecfg/lib/config/type/augeas ONECFG_LIB_CONFIG_TYPE_YAML_FILES:$LIB_LOCATION/onecfg/lib/config/type/yaml - ONECFG_SHARE_AUGEAS_FILES:$SHARE_LOCATION/onecfg/augeas + ONECFG_LIB_PATCH_FILES:$LIB_LOCATION/onecfg/lib/patch ONECFG_SHARE_ETC_FILES:$SHARE_LOCATION/onecfg/etc ) @@ -2706,11 +2706,15 @@ ONECFG_LIB_FILES="src/onecfg/lib/onecfg.rb src/onecfg/lib/config.rb \ src/onecfg/lib/exception.rb \ src/onecfg/lib/settings.rb \ + src/onecfg/lib/transaction.rb \ + src/onecfg/lib/patch.rb \ src/onecfg/lib/version.rb" -ONECFG_LIB_COMMON_FILES="src/onecfg/lib/common/backup.rb" +ONECFG_LIB_COMMON_FILES="src/onecfg/lib/common/backup.rb \ + src/onecfg/lib/common/parser.rb" ONECFG_LIB_COMMON_HELPERS_FILES="src/onecfg/lib/common/helpers/onecfg_helper.rb" ONECFG_LIB_COMMON_LOGGER_FILES="src/onecfg/lib/common/logger/cli_logger.rb" ONECFG_LIB_CONFIG_FILES="src/onecfg/lib/config/exception.rb \ + src/onecfg/lib/config/files.rb \ src/onecfg/lib/config/fsops.rb \ src/onecfg/lib/config/type.rb \ src/onecfg/lib/config/utils.rb" @@ -2719,11 +2723,10 @@ ONECFG_LIB_CONFIG_TYPE_FILES="src/onecfg/lib/config/type/augeas.rb \ src/onecfg/lib/config/type/simple.rb \ src/onecfg/lib/config/type/yaml.rb" ONECFG_LIB_CONFIG_TYPE_AUGEAS_FILES="src/onecfg/lib/config/type/augeas/one.rb \ - src/onecfg/lib/config/type/augeas/shell.rb" + src/onecfg/lib/config/type/augeas/shell.rb" ONECFG_LIB_CONFIG_TYPE_YAML_FILES="src/onecfg/lib/config/type/yaml/strict.rb" +ONECFG_LIB_PATCH_FILES="src/onecfg/lib/patch/apply.rb" -ONECFG_SHARE_AUGEAS_FILES="src/onecfg/share/augeas/oned.aug \ - src/onecfg/share/augeas/test_oned.aug" ONECFG_SHARE_ETC_FILES="src/onecfg/share/etc/files.yaml" diff --git a/share/install_gems/Gemfile b/share/install_gems/Gemfile index e1dc53c4c8..a0c763d16e 100644 --- a/share/install_gems/Gemfile +++ b/share/install_gems/Gemfile @@ -63,7 +63,6 @@ gem 'nokogiri', nokogiri gem 'public_suffix', ps group :cli do - gem 'augeas', '~> 0.6' gem 'gnuplot' gem 'highline', '~> 1.7' gem 'mysql2' @@ -72,6 +71,18 @@ group :cli do gem 'sequel' end +group :onecfg, :cli do + gem 'augeas', '~> 0.6' +end + +group :onecfg, :cloud, :oneflow, :sunstone do + gem 'json', '>= 2.0' +end + +group :onecfg do + gem 'git', '~> 1.5' +end + group :hybrid do gem 'aws-sdk-ec2', '>=1.151' gem 'aws-sdk-s3' @@ -94,7 +105,6 @@ end group :cloud, :oneflow, :sunstone do gem 'rack', rack - gem 'json' gem 'sinatra' end @@ -142,8 +152,3 @@ group :vmware do gem 'rbvmomi', '~> 2.2.0' end end - -group :onecfg do - gem 'git', '~> 1.5' - gem 'augeas', '~> 0.6' -end diff --git a/src/onecfg/bin/onecfg b/src/onecfg/bin/onecfg index 3d88ae5fb2..f3aa1875a2 100755 --- a/src/onecfg/bin/onecfg +++ b/src/onecfg/bin/onecfg @@ -41,6 +41,10 @@ $LOAD_PATH << LIB_LOCATION + '/onecfg/lib' # Supported patch modes SUPPORTED_PATCH_MODES = [:skip, :force, :replace] +# Early load of newer JSON gem, which supports serialization of scalars, e.g. +# > JSON.generate('string') +gem 'json', '>= 2.0' + require 'git' require 'tmpdir' require 'yaml' @@ -56,11 +60,143 @@ CommandParser::CmdParser.new(ARGV) do # onecfg helper helper = OneCfgHelper.new - # public command should go here + ######################################################################## + # Global Options + ######################################################################## + + UNPRIVILEGED = { + :name => 'unprivileged', + :large => '--unprivileged', + :description => 'Skip privileged operations (e.g., chown)' + } + + NO_OPERATION = { + :name => 'noop', + :short => '-n', + :large => '--noop', + :description => 'Runs update without changing system state' + } + + PREFIX = { + :name => 'prefix', + :large => '--prefix prefix', + :description => 'Root location prefix (default: /)', + :format => String + } + + ######################################################################## + # Logging modes + ######################################################################## + + VERBOSE = { + :name => 'verbose', + :short => '-d', + :large => '--verbose', + :description => 'Set verbose logging mode' + } + + DEBUG = { + :name => 'debug', + :large => '--debug', + :description => 'Set debug logging mode' + } + + DDEBUG = { + :name => 'ddebug', + :large => '--ddebug', + :description => 'Set extra debug logging mode' + } + + DDDEBUG = { + :name => 'dddebug', + :large => '--dddebug', + :description => 'Set extra debug logging mode' + } + + # logging modes + LOG_MODES = [VERBOSE, DEBUG, DDEBUG, DDDEBUG] + + ######################################################################## + # Command Specific Parameters + ######################################################################## + + PATCH_ALL = { + :name => 'all', + :short => '-a', + :large => '--all', + :description => 'All changes must be applied or patch fails' + } + + PATCH_FORMAT = { + :name => 'format', + :large => '--format format', + :description => 'Specify the patch input format. ' \ + 'Supported values are:' \ + ' "line" (single line format),' \ + ' "yaml" (YAML format).', + :format => String + } + + PATCH_FORMATS = %w[line yaml] begin require 'ee' OneCfg::EE::Commands.load_commands(self, ARGV, helper) rescue LoadError + # If we can't load EE, we are running CE only + end + + ########################################################################### + # Community Edition Commands + ########################################################################### + + patch_desc = <<-EOT.unindent + Apply changes to configuration files + EOT + + command :patch, patch_desc, + [:file, nil], + :options => [UNPRIVILEGED, + NO_OPERATION, + PREFIX, + PATCH_FORMAT, + PATCH_ALL] + + CommandParser::OPTIONS + LOG_MODES \ + do + OneCfg::LOG.get_logger(options) + rc = -1 + + begin + format = options[:format] unless options[:format].nil? + + if !format.nil? && !PATCH_FORMATS.include?(format) + STDERR.puts "Unsupported format '#{format}'. "\ + "Available formats - #{PATCH_FORMATS.join(', ')}" + exit(-1) + end + + patcher = OneCfg::Patch::Apply.new(options) + + if ARGV[0] + patcher.read_from_file(ARGV[0], format) + elsif !STDIN.tty? + Tempfile.open('stdin_patch_content') do |temp_file| + temp_file.write($stdin.read) + temp_file.close + + patcher.read_from_file(temp_file.path, format) + end + else + STDERR.puts 'No patch file or data on standard input found.' + exit(-1) + end + + rc = patcher.apply(options.key?(:all)) + rescue StandardError => e + OneCfg::LOG.fatal("FAILED - #{e}") + rc = -1 + end + + exit(rc) end end diff --git a/src/onecfg/lib/common.rb b/src/onecfg/lib/common.rb index e23dce5c71..0995f40dca 100644 --- a/src/onecfg/lib/common.rb +++ b/src/onecfg/lib/common.rb @@ -25,3 +25,4 @@ end require 'common/backup' require 'common/helpers/onecfg_helper' require 'common/logger/cli_logger' +require 'common/parser' diff --git a/src/onecfg/lib/common/backup.rb b/src/onecfg/lib/common/backup.rb index 13d506da71..c73157b85d 100644 --- a/src/onecfg/lib/common/backup.rb +++ b/src/onecfg/lib/common/backup.rb @@ -92,9 +92,11 @@ module OneCfg::Common backup(src, dst) end - # TODO: hmmm - versions = OneCfg::EE::Config::Versions.new - cfg_version = versions.cfg_version + begin + versions = OneCfg::EE::Config::Versions.new + cfg_version = versions.cfg_version + rescue NameError + end if cfg_version File.open("#{backup}/version", 'w') do |file| diff --git a/src/onecfg/lib/common/logger/cli_logger.rb b/src/onecfg/lib/common/logger/cli_logger.rb index 8fa0bae946..20bb31dd1c 100644 --- a/src/onecfg/lib/common/logger/cli_logger.rb +++ b/src/onecfg/lib/common/logger/cli_logger.rb @@ -75,7 +75,8 @@ module OneCfg # # @param msg [String] Message to show def self.ddebug(msg) - return unless instance.custom_level == :ddebug + return unless \ + [:ddebug, :dddebug].include?(instance.custom_level) instance.logger.debug(msg) # TODO end diff --git a/src/onecfg/lib/common/parser.rb b/src/onecfg/lib/common/parser.rb new file mode 100644 index 0000000000..584a6285b2 --- /dev/null +++ b/src/onecfg/lib/common/parser.rb @@ -0,0 +1,155 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'strscan' + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Common + + # HintingParser class + class HintingParser + + def initialize(string) + @scanner = StringScanner.new(string) + end + + # Parse string passed to constructor as single hinting in a format + # [] + # and return hash with :command, :path, :value keys. + # + # @param normalize [Bool] Check and process parsed values + # + # @return [Hash] Hash with parsed data + def parse(normalize = true) + # reset pointer to the begining + @scanner.pointer = 0 + + ret = { + :command => get_word, + :path => get_word, + :value => get_until_end + } + + normalize!(ret) if normalize + + ret + end + + # Parse string passed to constructor as single hinting in a format + # [] + # and return hash with :filename, :command, :path, :value keys. + # + # @param normalize [Bool] Check and process parsed values + # + # @return [Hash] Hash with parsed data + def parse_with_filename(normalize = true) + # reset pointer to the beginning + @scanner.pointer = 0 + + ret = { + :filename => get_word, + :command => get_word, + :path => get_word, + :value => get_until_end + } + + normalize!(ret) if normalize + + ret + end + + private + + QUOTES_STRING = ['"'] + + # Validate and normalize parser data. Works on input data. + # + # @param parsed [Hash] parsed data + # + # @return [Hash] normalized parsed data + def normalize!(parsed) + # command + parsed[:command].downcase! + + if !%w[rm ins set].include?(parsed[:command]) + raise OneCfg::Exception::FileParseError, + "Invalid patch action '#{parsed[:command]}'" + end + + # path + parsed[:path] = parsed[:path].split('/').reject(&:empty?) + + # value + unless parsed[:value].nil? + parsed[:value] = JSON.parse(parsed[:value]) + end + + parsed + end + + # rubocop:disable Naming/AccessorMethodName + # TODO, uncovered case: + # - When get word it's called for strings like: path/\"kvm \"/path, + # it won't parse the scaped quotes in the middle of the word. + # Workaround: use single quotes 'path/path/"kvm "/path' + def get_word + return if @scanner.eos? + + # advance until first char difference from space + if @scanner.match?(/\s/) + @scanner.scan_until(/\S/) + @scanner.pointer -= 1 + end + + if QUOTES_STRING.include? @scanner.peek(1) + # read until next quote + match = @scanner.scan(/(?:"(?[^"\\]*(?:\\.[^"\\]*)*"))/) + + # remove last quote and unscape them + match = match.strip[1..-2].gsub('\"', '"') unless match.nil? + else + match = @scanner.scan_until(/\s/) + if match.nil? + match = @scanner.scan_until(/$/) + end + + match.strip! unless match.nil? + end + + # Advanced until next word + @scanner.scan(/\s+/) + + match + end + + def get_until_end + return if @scanner.eos? + + match = @scanner.scan_until(/$/) + + if !match.nil? + # unscape elements + match = match.strip + end + + match + end + # rubocop:enable Naming/AccessorMethodName + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config.rb b/src/onecfg/lib/config.rb index 17f5d53304..d62cd35900 100644 --- a/src/onecfg/lib/config.rb +++ b/src/onecfg/lib/config.rb @@ -26,6 +26,7 @@ require 'config/type/yaml/strict' require 'config/utils' require 'config/exception' require 'config/fsops' +require 'config/files' begin require 'ee/config' diff --git a/src/onecfg/lib/config/exception.rb b/src/onecfg/lib/config/exception.rb index 4f04d678c8..84401d1c9e 100644 --- a/src/onecfg/lib/config/exception.rb +++ b/src/onecfg/lib/config/exception.rb @@ -159,6 +159,17 @@ module OneCfg::Config::Exception end + # Exception to indicate that the current patch + # operation should be restarted by searching + # the place in a tree and reapplication. + class PatchRetryOperation < PatchException + + def initialize + super('Retrying patch operation') + end + + end + end # rubocop:enable Style/ClassAndModuleChildren # rubocop:enable Lint/UselessMethodDefinition diff --git a/src/onecfg/lib/config/files.rb b/src/onecfg/lib/config/files.rb new file mode 100644 index 0000000000..066bb6323d --- /dev/null +++ b/src/onecfg/lib/config/files.rb @@ -0,0 +1,379 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Config + + # Class to scan and validate configuration files + class Files < OneCfg::Settings + + # Base name prefix for configuration classes + TYPE_CLASS_NAME = 'OneCfg::Config::Type' + + # Class constructor + def initialize + super(OneCfg::FILES_CFG, [:auto_load, :read_only]) + + # we really need the classification data + # rubocop:disable Style/GuardClause + if !@content.is_a?(Array) || @content.empty? + raise OneCfg::Config::Exception::FatalError, + 'Missing classification of configuration files' + end + # rubocop:enable Style/GuardClause + end + + def scan(prefix = '/', strict = true) + ret = {} + + # fileops provides sandboxed operations + fops = OneCfg::Config::FileOperation.new(prefix) + + @content.each do |file| + fops.glob(file['name'], false).each do |match| + next if fops.directory?(match) + + if file.key?('class') + # file already scanned matched once again + if ret.key?(match) + raise OneCfg::Config::Exception::FatalError, + "File '#{match}' already scanned. " \ + 'Duplicate expressions matching same file.' + else + ret[match] = file.dup + + # replace stripped class with real Ruby class + begin + ret[match]['ruby_class'] = \ + self.class.type_class(file['class']) + rescue NameError + raise OneCfg::Config::Exception::FatalError, + "File '#{match}' has invalid class " \ + "'#{file['class']}'" + end + end + + # fail in strict mode if we found some files + # matched by catch-all expressions + elsif strict && !ret.key?(match) + raise OneCfg::Config::Exception::FatalError, + "File '#{match}' doesn't have classification." + end + end + end + + ret + end + + # Validates all known files from specified prefix can be read and + # and written into temporary location. Checks that files and + # OneCfg code is OK. + # + # @param prefix [String] Root prefix + # + # @return [Boolean] True if all valid + def validate(prefix = '/') + ret = true + + scan(prefix, false).each do |name, data| + begin + # create type object and load + pre_name = OneCfg::Config::Utils.prefixed(name, prefix) + + OneCfg::LOG.debug("Load '#{pre_name}' " \ + "with #{data['ruby_class']}") + + file = data['ruby_class'].new(pre_name) + file.load + + # valid file and OneCfg code must evaluate file + # as same, similar and without diff on self + unless file.same?(file) && + file.similar?(file) && + file.diff(file).nil? + raise OneCfg::Config::Exception::StructureError, + 'Error when comparing file content with self' + end + + # test save into temporary file + Tempfile.open('validate') do |temp| + OneCfg::LOG.debug("Save '#{pre_name}' " \ + "into '#{temp.path}'") + + temp.close + file.save(temp.path) + end + + OneCfg::LOG.info("File '#{pre_name}' - OK") + rescue StandardError => e + ret = false + + # if any error, just print it and continue with the rest + OneCfg::LOG.error('Unable to process file ' \ + "'#{name}' - #{e.message}") + end + end + + ret + end + + # Show hintings based on diffs + # + # @param prefix [String] Path to OpenNebula installation + # + # @return [String] Returns string with diff + def diff(prefix, diff_format) + versions = OneCfg::EE::Config::Versions.new + migrators = versions.migrators + migrators = migrators.get_migrators_range(migrators.all_from, + versions.cfg_version) + migrators.select! {|m| m.yaml? } + + migrator = migrators.last + + # load YAML descriptor + if migrator.nil? + raise OneCfg::Config::Exception::UnsupportedVersion, + 'Could not find suitable migrator with files' + end + + migrator.load_yaml + + if !migrator.yaml || !migrator.yaml['patches'] + # this should not happen + raise OneCfg::Config::Exception::FatalError, + "Migrator for #{migrator.label} doesn't have valid YAML" + end + + ### + + OneCfg::LOG.debug('Comparing against state from ' \ + "#{migrator.label} migrator") + + patch_gen = OneCfg::EE::Patch::Generate.new(prefix) + + # process each file from descriptor + migrator.yaml['patches'].each do |name, info| + patch_gen.add(name, info) + end + + case diff_format + when 'yaml' + patch_gen.generate_yaml + when 'line' + patch_gen.generate_line + else + patch_gen.generate_hintings + end + end + + # Get configuration type class from stripped string. + # + # @param [String] Stripped class name + # + # @return [Class] Ruby class + def self.type_class(name) + Kernel.const_get("#{TYPE_CLASS_NAME}::#{name}") + end + + # Filters single file metadata entry returned by scan function + # for suitable keys, which can be used for generated YAML migrator. + # + # @param [Hash] File data + # + # @return [Hash] Filtered file data + def file4desc(data) + data.select do |k, _v| + %w[class owner group mode].include?(k) + end + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren + +__END__ + + def cfg_version + ret = nil + + @settings.load + + if @settings.content && @settings.content['version'] + ret = Gem::Version.new(@settings.content['version']) + end + + ret + rescue StandardError + nil + end + + def cfg_version=(version) + unless version + raise OneCfg::Config::Exception::FatalError, + 'Missing configuration version to save' + end + + # TODO: can fail? + @settings.load + @settings.content['version'] = version.to_s + @settings.save + end + + # Return installed OpenNebula version + # + # @param prefix [String] Location prefix + # + # @return [Gem::Version, Nil] ) Gem::Version or Nil for unknown. + def one_version(prefix = nil) + ret = nil + + if prefix + cmd = "#{prefix}/bin/oned --version" + else + cmd = 'oned --version' + end + + o, _e, s = Open3.capture3(cmd) + + if s.success? + matches = o.match(/^OpenNebula (\d+\.\d+.\d+) /) + + if matches + ret = Gem::Version.new(matches[1]) + end + end + + # TODO: add alternative methods (e.g., query packager) + + ret + rescue StandardError + nil + end + + # Return first defined version. + def defaults(ver1, ver2) + # take first defined version + ret = ver1 + ret ||= ver2 + ret = Gem::Version.new(ret) if ret && !ret.is_a?(Gem::Version) + + unless ret + raise OneCfg::Config::Exception::UnsupportedVersion, + 'Unknown OpenNebula or config. version. Check status.' + end + + # check version is supported by migrators + unless @migrators.supported?(ret) + raise OneCfg::Config::Exception::UnsupportedVersion, + "Unsupported version #{ret}" + end + + ret + end + + # Dumps versions status for poor humans beings + # + # @return[[String, Integer]] Returns twin with String and Integer. + # String is a text to show to the user. + # Integer is the exit status. + def dump_status + ret = '' + + ret << "--- Versions -----------------\n" + + ret << format("%-11s %-8s\n", + 'OpenNebula:', + one_version ? one_version : 'unknown') + + ret << format("%-11s %-8s\n", + 'Config:', + cfg_version ? cfg_version : 'unknown') + + unless one_version + ret << "ERROR: Unknown OpenNebula version\n" + return([ret, -1]) + end + + if !supported?(one_version) + ret << "ERROR: Unsupported OpenNebula version #{one_version}\n" + return([ret, -1]) + end + + unless cfg_version + ret << "ERROR: Unknown config version\n" + return([ret, -1]) + end + + if !supported?(cfg_version) + ret << "ERROR: Unsupported config version #{cfg_version}\n" + return([ret, -1]) + end + + ret << "\n--- Available updates --------\n" + + if cfg_version && latest && latest > cfg_version + ret << format("%-11s %-8s\n", 'New config:', latest) + end + + # TODO: test for downgrade + if upgrades? + get_migrators.each do |u| + ret << "- from #{u.from} to #{u.to} (" + ret << 'YAML' if u.yaml? + ret << ',' if u.yaml? && u.ruby? + ret << 'Ruby' if u.ruby? + ret << ")\n" + end + + return([ret, 1]) + elsif cfg_version && latest && latest > cfg_version + # TODO: how this can happen? + ret << "No updates available, but config. is not latest!?!\n" + else + ret << "No updates available.\n" + end + + [ret, 0] + end + + ### Proxy methods into migrators ### + + # Gets latest (highest) configuration version matching + # current OpenNebula version. + def latest(v = one_version) + @migrators.latest(v) + end + + # Check if there is any update. + def upgrades?(v1 = cfg_version, v2 = one_version) + @migrators.upgrades?(v1, v2) + end + + # Check if version is supported + def supported?(version) + @migrators.supported?(version) + end + + def get_migrators(v1 = cfg_version, v2 = one_version) + @migrators.get_migrators_range(v1, v2) + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/config/type/augeas/one.rb b/src/onecfg/lib/config/type/augeas/one.rb index bd723f9b43..6a70589521 100644 --- a/src/onecfg/lib/config/type/augeas/one.rb +++ b/src/onecfg/lib/config/type/augeas/one.rb @@ -63,7 +63,7 @@ module OneCfg::Config::Type # # @param name [String] File name # @param load_path [String] directories for modules - def initialize(name = nil, load_path = OneCfg::LENS_DIR) + def initialize(name = nil, load_path = nil) super(name, 'Oned.lns', load_path) end @@ -721,7 +721,7 @@ module OneCfg::Config::Type if found.empty? ret[:status] = true - content.set(path, data['value']) + content.set(path, data['value'].to_s) else is_m = multiple?([data['path'], data['key']]) @@ -733,13 +733,13 @@ module OneCfg::Config::Type if is_m unless found.include?(data['value']) - content.set("#{path}[0]", data['value']) + content.set("#{path}[0]", data['value'].to_s) ret[:status] = true end elsif found.length == 1 if found[0] != data['value'] && mode.include?(:replace) - content.set(path, data['value']) + content.set(path, data['value'].to_s) ret[:status] = true ret[:mode] = :replace @@ -755,7 +755,7 @@ module OneCfg::Config::Type content.rm("#{path}[last()]") end - content.set(path, data['value']) + content.set(path, data['value'].to_s) ret[:status] = true ret[:mode] = :replace @@ -773,7 +773,7 @@ module OneCfg::Config::Type ret end - # Appply single diff/patch "set" operation. + # Apply single diff/patch "set" operation. # # @param path [String] Augeas path # @param data [Hash] Single diff operation data @@ -792,9 +792,9 @@ module OneCfg::Config::Type # can be multiple params created wrongly by user. So we # try to test and set only first one. Nothing else # matters anyway... - content.set("#{path}[1]", data['value']) + content.set("#{path}[1]", data['value'].to_s) else - content.set(path, data['value']) + content.set(path, data['value'].to_s) end ret[:status] = true @@ -806,7 +806,7 @@ module OneCfg::Config::Type end end - content.set(path, data['value']) + content.set(path, data['value'].to_s) ret[:status] = true ret[:mode] = :replace @@ -825,6 +825,11 @@ module OneCfg::Config::Type # # @return [String] Formatted key def hinting_key(data) + return super(data) + + ### This function is disabled for now ### + # rubocop:disable Lint/UnreachableCode + full_path = [data['path'], data['key']].flatten.compact if full_path.empty? @@ -843,6 +848,7 @@ module OneCfg::Config::Type full_path.join('/') end + # rubocop:enable Lint/UnreachableCode end # Upcase node names in the Augeas content object to avoid diff --git a/src/onecfg/lib/config/type/augeas/shell.rb b/src/onecfg/lib/config/type/augeas/shell.rb index 6f4fab781f..23be3519b3 100644 --- a/src/onecfg/lib/config/type/augeas/shell.rb +++ b/src/onecfg/lib/config/type/augeas/shell.rb @@ -311,7 +311,7 @@ module OneCfg::Config::Type # New variables are easily set with required export status. if found.empty? - content.set(key, val) + content.set(key, val.to_s) export_key(key, exp) ret[:status] = true @@ -324,7 +324,7 @@ module OneCfg::Config::Type content.rm("#{key}[1]") end - content.set(key, val) + content.set(key, val.to_s) export_key(key, exp) ret[:status] = true @@ -335,7 +335,7 @@ module OneCfg::Config::Type ret end - # Appply single diff/patch "set" operation. + # Apply single diff/patch "set" operation. # # @param data [Hash] Single diff operation data # @param mode [Array] Patch modes (see patch method) @@ -358,7 +358,7 @@ module OneCfg::Config::Type content.rm("#{key}[1]") end - content.set(key, val) + content.set(key, val.to_s) ret[:status] = true ret[:mode] = :replace if found[-1] != old @@ -381,29 +381,17 @@ module OneCfg::Config::Type ret end - # Get value for hintings - # - # @param data [Hash] Element with diff information - # - # @return [String] Formatted value - def hinting_value(data) - ret = nil - - if data.key?('value') - ret = data['value'].inspect - elsif data.key?('old') - ret = data['old'].inspect - end - - ret - end - # Get extra metadata for hintings # # @param data [Hash] Element with diff information # # @return [String] Formatted value def hinting_extra(data) + return super(data) + + ### This function is disabled for now ### + # rubocop:disable Lint/UnreachableCode + return unless data.key?('extra') ret = '' @@ -417,6 +405,7 @@ module OneCfg::Config::Type end ret.strip + # rubocop:enable Lint/UnreachableCode end # Returns all details from single diff operation metadata diff --git a/src/onecfg/lib/config/type/base.rb b/src/onecfg/lib/config/type/base.rb index dc32bb9773..8180158db8 100644 --- a/src/onecfg/lib/config/type/base.rb +++ b/src/onecfg/lib/config/type/base.rb @@ -17,6 +17,11 @@ require 'fileutils' require 'open3' +# Early load of newer JSON gem, which supports serialization of scalars, e.g. +# > JSON.generate('string') +gem 'json', '>= 2.0' +require 'json' + # rubocop:disable Style/ClassAndModuleChildren module OneCfg::Config::Type @@ -165,7 +170,7 @@ module OneCfg::Config::Type # # @param cfg[OneCfg::Config::Base] Configuration to diff with # - # @return [String, nil] String with diff if files are not + # @return [Array, nil] Array with diff if files are not # identical. nil if files are identical. Exception on error. def diff(cfg) data = file_diff(cfg) @@ -184,6 +189,71 @@ module OneCfg::Config::Type end end + # Create a diff metadata based based on hintings. + # + # @param diff [Array] Array of strings with hintings + # @param symbols [Bool] If symbol-like strings should be symbolized + # + # @return [Array] Array with diff. + def diff_from_hintings(hintings, symbols = false) + # hinting formats: + # - [] + parsed = OneCfg::Common::HintingParser.new(hintings).parse + + # path/key value + state = parsed[:command] + path = parsed[:path] + key = path.pop + value = parsed[:value] + + # symbolize path/key/value back + if symbols + path = symbolize(path) + key = symbolize(key) + value = symbolize(value) + end + + # TODO, check how to manage extras + extra = { 'hintings' => true } + + ret = { + 'state' => state, + 'path' => path, + 'key' => key, + 'value' => nil, + 'old' => nil, + 'extra' => extra + } + + case state + when 'rm' + ret['old'] = value + + # Problematic case: on removal of array element, the path + # contains whole path to the array, but there is no key, + # since it's not a key/valued-hash. We identify array + # element removal by presence of value in rm command. + # + # > rm path/key "value" + # + # In this case, we put key back into path structure. + unless value.nil? || key.nil? + ret['path'] << key + ret['key'] = nil + end + when 'set' + # 'set' changes existing value from old to value. This + # workarounds the fact that we don't know old value and + # forces the change to happen if target is not on value. + ret['value'] = value + ret['old'] = value + else + ret['value'] = value + end + + ret + end + # Patches object based on provided diff data. # # @param data [Array] Diff data @@ -236,6 +306,8 @@ module OneCfg::Config::Type ret ||= patch_status[:status] rep << patch_status + rescue OneCfg::Config::Exception::PatchRetryOperation + retry # TODO: rescue on any exception rescue StandardError => e @@ -286,18 +358,18 @@ module OneCfg::Config::Type case d['state'] when 'ins' - ret << "#{r_status}ins #{key} = #{value} #{extra}" + ret << "#{r_status}ins #{key} #{value} #{extra}" when 'set' if value.nil? ret << "#{r_status}set #{key} #{extra}" else - ret << "#{r_status}set #{key} = #{value} #{extra}" + ret << "#{r_status}set #{key} #{value} #{extra}" end when 'rm' if value.nil? ret << "#{r_status}rm #{key} #{extra}" else - ret << "#{r_status}rm #{key} = #{value} #{extra}" + ret << "#{r_status}rm #{key} #{value} #{extra}" end else ret << "unknown operation #{state}" @@ -398,36 +470,91 @@ module OneCfg::Config::Type ret end + # Looks into value and all symbols convers to strings starting with ':'. + # Recursively goes deep through the arrays and hashes, where + # stringifies both suspicious keys and values. + # + # @param value [Any] Value to stringify + # + # @return [Any] Value with symbols converted to strings starting with : + def unsymbolize(value) + # rubocop:disable Style/CaseLikeIf + if value.is_a?(Symbol) + ":#{value}" + elsif value.is_a?(Array) + value.collect {|v| unsymbolize(v) } + elsif value.is_a?(Hash) + Hash[value.collect {|k, v| [unsymbolize(k), unsymbolize(v)] }] + else + value + end + # rubocop:enable Style/CaseLikeIf + end + + # Looks into value and texts which begin with ':' converts to symbols. + # Recursively goes deep through the arrays and hashes, where symbolizes + # back both suspicious keys and values. + # + # @param value [Any] Value to symbolize + # + # @return [Any] Value with symbolized values where begins with : + def symbolize(value) + # TODO: don't symbolize value if it contains spaces? + if value.is_a?(String) && value.start_with?(':') && value.length > 1 + value[1..-1].to_sym + elsif value.is_a?(Array) + value.collect {|v| symbolize(v) } + elsif value.is_a?(Hash) + Hash[value.collect {|k, v| [symbolize(k), symbolize(v)] }] + else + value + end + end + # Get key for hintings # - # @param diff [Hash] Element with diff information + # @param diff [Hash] Element with diff information + # @param symbols [Bool] If symbols in paths should be preserved # # @return [String] Formatted key - def hinting_key(data) + def hinting_key(data, symbols = false) full_path = [data['path'], data['key']].flatten.compact if full_path.empty? - '(top)' + ret = '/' + elsif symbols + ret = unsymbolize(full_path).join('/') else - full_path.join('/') + ret = full_path.join('/') end + + # if there are special characters, better quote + # TODO: parser probably doesn't handle correctly \'?? + ret.index(/[" ]/) ? ret.inspect : ret end # Get value for hintings # - # @param data [Hash] Element with diff information + # @param data [Hash] Element with diff information + # @param symbols [Bool] If symbols in values should be preserved # # @return [String] Formatted value - def hinting_value(data) - ret = '??? UNKNOWN ???' + def hinting_value(data, symbols = false) + ret = nil if data.key?('value') - ret = data['value'].inspect - elsif data.key?('old') - ret = data['old'].inspect + ret = data['value'] + elsif data.key?('old') && !data['key'] + ret = data['old'] + else + # Quiet here, just in case we are going to JSONify nil, + # as it can be value if set by above cases!! + return '' end - ret + ret = unsymbolize(ret) if symbols + + JSON.generate(ret) end # Get extra metadata for hintings diff --git a/src/onecfg/lib/config/type/yaml.rb b/src/onecfg/lib/config/type/yaml.rb index 92f582dcef..ca38aed593 100644 --- a/src/onecfg/lib/config/type/yaml.rb +++ b/src/onecfg/lib/config/type/yaml.rb @@ -85,12 +85,24 @@ module OneCfg::Config::Type ret.empty? ? nil : ret end + def diff_from_hintings(hintings) + super(hintings, true) + end + ################################################################## # Private Methods ################################################################## private + def hinting_key(data) + super(data, true) + end + + def hinting_value(data) + super(data, true) + end + # Walks through the tree1 and tree2 data structures # starting the provided path, compares them and returns # array of differences. @@ -476,6 +488,24 @@ module OneCfg::Config::Type # insert new key into a Hash if data['key'] + + ### WORKAROUND + # Insert action created from hintings can't properly + # distinguish between insert into Array and Hash and + # set path/key properly. We double check if we don't + # try to insert into actual Array and if yes, we + # update the diff structure and retry operation. + # rubocop:disable Style/SoleNestedConditional + if data['extra'] && data['extra']['hintings'] + if tree.is_a?(Hash) && tree[data['key']].is_a?(Array) + data['path'] << data['key'] + data['key'] = nil + + raise OneCfg::Config::Exception::PatchRetryOperation + end + end + # rubocop:enable Style/SoleNestedConditional + if tree.is_a? Hash if tree.key?(data['key']) dc = OneCfg::Config::Utils.deep_compare( diff --git a/src/onecfg/lib/exception.rb b/src/onecfg/lib/exception.rb index 1c0fb29258..b0c646024e 100644 --- a/src/onecfg/lib/exception.rb +++ b/src/onecfg/lib/exception.rb @@ -59,6 +59,17 @@ module OneCfg end # rubocop:enable Lint/UselessMethodDefinition + # OneCfg parser exception on file + # rubocop:disable Lint/UselessMethodDefinition + class FileParseError < Generic + + def initialize(text) + super(text) + end + + end + # rubocop:enable Lint/UselessMethodDefinition + end end diff --git a/src/onecfg/lib/onecfg.rb b/src/onecfg/lib/onecfg.rb index 46c582fe91..fa87cdad6e 100644 --- a/src/onecfg/lib/onecfg.rb +++ b/src/onecfg/lib/onecfg.rb @@ -19,24 +19,31 @@ require 'common' require 'settings' require 'version' require 'config' +require 'transaction' +require 'patch' begin require 'ee' rescue LoadError + # If we can't load EE, we are running CE only end # OneCfg main module module OneCfg + LOG = OneCfg::Common::CliLogger + ONE_LOCATION = ENV['ONE_LOCATION'] + ETC_DIR = ENV['ONECFG_ETC_DIR'] if ENV.key?('ONECFG_ETC_DIR') + BACKUP_DIR = ENV['ONECFG_BACKUP_DIR'] if ENV.key?('ONECFG_BACKUP_DIR') # Global directories # TODO: improve if ONE_LOCATION BIN_DIR = File.join(ONE_LOCATION, 'bin') - ETC_DIR = '/tmp/onescape/etc' - BACKUP_DIR = '/tmp/onescape/backups' SHARE_DIR = ONE_LOCATION + '/share/onecfg' + ETC_DIR ||= '/tmp/onescape/etc' + BACKUP_DIR ||= '/tmp/onescape/backups' [ETC_DIR, BACKUP_DIR].each do |d| OneCfg::LOG.warn("Using local state in #{d}") @@ -44,13 +51,11 @@ module OneCfg end else BIN_DIR = '/usr/bin' - ETC_DIR = '/etc/onescape' - BACKUP_DIR = '/var/lib/one/backups/config' - SHARE_DIR = '/usr/share/one/onecfg' + SHARE_DIR = '/usr/share/one/onecfg' + ETC_DIR ||= '/etc' + BACKUP_DIR ||= '/var/lib/one/backups/config' end - LOG = OneCfg::Common::CliLogger - # Project local directories CONF_DIR = File.join(SHARE_DIR, 'etc') MIGR_DIR = File.join(SHARE_DIR, 'migrators') @@ -58,7 +63,7 @@ module OneCfg # Individual files FILES_CFG = File.join(CONF_DIR, 'files.yaml') - CONFIG_CFG = File.join(ETC_DIR, 'config.yaml') + CONFIG_CFG = File.join(ETC_DIR, 'onecfg.conf') # Configuration management releated constants CONFIG_BACKUP_DIRS = ['/etc/one', '/var/lib/one/remotes'] diff --git a/src/onecfg/lib/patch.rb b/src/onecfg/lib/patch.rb new file mode 100644 index 0000000000..b09f4cbe53 --- /dev/null +++ b/src/onecfg/lib/patch.rb @@ -0,0 +1,31 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require_relative 'patch/apply' + +begin + require 'ee/patch' +rescue LoadError +end + +module OneCfg + + # OneCfg::Patch module + module Patch + + end + +end diff --git a/src/onecfg/lib/patch/apply.rb b/src/onecfg/lib/patch/apply.rb new file mode 100644 index 0000000000..840166b009 --- /dev/null +++ b/src/onecfg/lib/patch/apply.rb @@ -0,0 +1,221 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# rubocop:disable Style/ClassAndModuleChildren +module OneCfg::Patch + + # Class for generating patches + class Apply + + # Class constructor + def initialize(args = {}) + @patches = { 'patches' => {} } + + # TODO: move common defaults on a single place + @prefix = '/' + @unprivileged = false + @no_operation = false + + # set based on args + @prefix = args[:prefix] unless args[:prefix].nil? + @unprivileged = args[:unprivileged] unless args[:unprivileged].nil? + @no_operation = args[:noop] unless args[:noop].nil? + end + + def read_from_file(filename, format) + if format.nil? + begin + if File.open(filename, &:readline).strip == '---' + format = 'yaml' + end + rescue StandardError + # we do silently ignore any errors in format detection + # as they might be much better handled below + end + + format ||= 'line' + end + + begin + case format + when 'yaml' + parse_yaml(filename) + else + parse_hintings(filename) + end + rescue OneCfg::Exception::FileParseError => e + raise e + rescue StandardError => e + raise "Error reading #{format} patch (#{e})" + end + end + + def apply(all = true) + if @patches['patches'].empty? + OneCfg::LOG.error('No changes to apply') + return(-1) + end + + # all changes are running in transaction, + # which is "rollbacked" in case of error + tr = OneCfg::Transaction.new + + tr.prefix = @prefix + tr.unprivileged = @unprivileged + tr.no_operation = @no_operation + + OneCfg::LOG.info('Applying patch to ' \ + "#{@patches['patches'].size} files") + + ret = 0 + + # total number of changes + total_count = 0 + total_success = 0 + + tr.execute do |tr_prefix, _fops| + @patches['patches'].each do |filename, patch| + # Get prefixed name (do not modified original file) + pre_name = OneCfg::Config::Utils.prefixed(filename, + tr_prefix) + + # Get file object based on the type + OneCfg::LOG.ddebug("Reading file '#{filename}'") + file = OneCfg::Config::Files.type_class(patch['class']) + .new(pre_name) + file.load + + # Apply changes + begin + OneCfg::LOG.debug("Patching file '#{filename}'") + rc, rep = file.patch(patch['change'], [:replace]) + rescue StandardError => e + OneCfg::LOG.error('Failed to patch file ' \ + "'#{filename}' - #{e}") + return(-1) + end + + # Count non/applied patches + patch_count = rep.size + patch_success = 0 + + rep.each.each do |r| + if r[:status] + patch_success += 1 + else + ret = 1 + end + end + + # Add counts to totals + total_count += patch_count + total_success += patch_success + + OneCfg::LOG.info("Patched '#{filename}' with " \ + "#{patch_success}/#{patch_count} " \ + 'changes') + + # show report + OneCfg::LOG.debug("--- PATCH REPORT '#{filename}' --- ") + file.hintings(patch['change'], rep).each do |l| + OneCfg::LOG.debug("Patch #{l}") + end + + file.save if rc + end + + # there were changes, but nothing could be applied + if total_success == 0 + OneCfg::LOG.error('No changes applied') + return(-1) + end + + # use requested all changes to apply, not even less + if all && total_success != total_count + OneCfg::LOG.error('Modifications not saved due to ' \ + "#{total_count - total_success} " \ + 'unapplied changes!') + return(-1) + end + + # statistics at the end before finishing transaction + OneCfg::LOG.info("Applied #{total_success}/#{total_count} " \ + 'changes') + end + + ret + end + + private + + # Add a file diff to be processed + # + # @param filename [String] path to patch in YAML format + def parse_yaml(filename) + @patches = YAML.load_file(filename) + + return if @patches.is_a?(Hash) + + raise OneCfg::Exception::FileParseError, + 'The patch does not contain expected YAML document' + end + + # Add a file diff to be processed in hinting format + # + # @param filename [String] path to patch in hintings format + def parse_hintings(filename) + files = OneCfg::Config::Files.new.scan(@prefix, false) + + changes = {} + + File.open(filename).each_with_index do |line, idx| + # skip empty lines and comments + line.strip! + next if line.empty? || line.start_with?('#') + + # line format: + if line.split.size < 2 + raise OneCfg::Exception::FileParseError, + "Invalid format at line: #{idx + 1}." + end + + # TODO: should be managed by parser + h_filename, hintings = line.split(' ', 2) + + unless files.key?(h_filename) + raise OneCfg::Exception::FileParseError, + "Unknown or missing file to patch '#{h_filename}'" + end + + # get suitable file manipulation object and create a diff + file_obj = files[h_filename]['ruby_class'].new(h_filename) + changes[h_filename] ||= [] + changes[h_filename] << file_obj.diff_from_hintings(hintings) + end + + # create internal structure similar to what we get from YAML + changes.each do |f, c| + @patches['patches'][f] = { + 'class' => files[f]['class'], + 'change' => c + } + end + end + + end + +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/src/onecfg/lib/transaction.rb b/src/onecfg/lib/transaction.rb new file mode 100644 index 0000000000..caddc03d96 --- /dev/null +++ b/src/onecfg/lib/transaction.rb @@ -0,0 +1,194 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +module OneCfg + + # Transactional operations with configuration files + class Transaction + + attr_accessor :prefix + attr_accessor :read_from + attr_accessor :unprivileged + attr_accessor :no_operation + attr_accessor :hook_post_copy + + def initialize + # TODO: move common defaults on a single place + @prefix = '/' + @read_from = nil + @unprivileged = false + @no_operation = false + @hook_post_copy = nil + end + + # Runs a passed code block on a copy of configuration files in + # a transaction-like way. If block finishes successfully, + # the changed configuration files are copied back to their + # right place. Code gets transactino prefix directory and + # file FileOperation object. + # + # @yield [tr_prefix, fops] Execute custom code with + # + # @return [Boolean,Nil] True on successful migration. + def execute + OneCfg::LOG.ddebug("Preparing transaction for prefix '#{@prefix}'") + + check_symlinks(@prefix) + check_symlinks(@read_from) if @read_from + + ### emergency backup ### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + backup = backup_dirs + ### emergency backup ### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + + tr_prefix = Dir.mktmpdir + + # copy data from @read_from/@prefix into transaction_prefix + # rubocop:disable Style/RedundantCondition + OneCfg::Common::Backup.restore_dirs( + @read_from ? @read_from : @prefix, + OneCfg::CONFIG_BACKUP_DIRS, + tr_prefix + ) + # rubocop:enable Style/RedundantCondition + + # file operations will be locked to transaction prefix + fops = OneCfg::Config::FileOperation.new(tr_prefix, @unprivileged) + + # run custom code + OneCfg::LOG.ddebug("Running transaction in '#{tr_prefix}'") + ret = yield(tr_prefix, fops) + + # in no-operation mode, we finish before copying + # changed configuration back + if @no_operation + OneCfg::LOG.ddebug('Transaction code successful, but ' \ + 'executed in no-op mode. Changes will ' \ + 'NOT BE SAVED!') + + OneCfg::LOG.info('Changes ARE NOT saved in no-op mode!') + + return(ret) + end + + OneCfg::LOG.ddebug('Transaction code successful') + + # Copy updated configuration back from transaction_prefix + # to original @prefix location. Restore on any failure. + begin + # We copy back from transaction_prefix only CONFIG_UPDATE_DIRS, + # which should be a subset of directories we have + # backuped. This enables to work with whole remotes/, + # but copy back only remotes/etc. + OneCfg::Common::Backup.restore_dirs( + tr_prefix, + OneCfg::CONFIG_UPDATE_DIRS, # <-- !!! + @prefix + ) + + if @hook_post_copy + OneCfg::LOG.ddebug('Running transaction post-copy hook') + @hook_post_copy.call(ret) + end + rescue StandardError + restore_dirs(backup) + + raise + end + + OneCfg::LOG.ddebug('Transaction done') + + ret + ensure + # cleanup temporary transaction directory + if defined?(tr_prefix) && !tr_prefix.nil? && File.exist?(tr_prefix) + FileUtils.rm_r(tr_prefix) + end + end + + private + + # Checks there are no symlinks in backup directories. + # Raise exception in case of error. + # + # @param custom_prefix [String] Custom prefix to check + def check_symlinks(custom_prefix = @prefix) + OneCfg::CONFIG_BACKUP_DIRS.each do |dir| + pre_dir = OneCfg::Config::Utils.prefixed(dir, custom_prefix) + OneCfg::LOG.dddebug("Checking symbolic links in '#{pre_dir}'") + + Dir["#{pre_dir}/**/**"].each do |f| + if File.symlink?(f) + raise OneCfg::Config::Exception::FatalError, + "Found symbolic links in '#{f}'" + end + end + end + end + + # Backup all dirs inside prefix. This is intended as + # backup for emergency cases, when upgrade fails and + # we need to revert changes back. + # + # @return [String] Backup path + def backup_dirs + backup = OneCfg::Common::Backup.backup_dirs( + OneCfg::CONFIG_BACKUP_DIRS, + nil, # backup name autogenerated + @prefix + ) + + OneCfg::LOG.unknown("Backup stored in '#{backup}'") + + backup + rescue StandardError => e + raise OneCfg::Config::Exception::FatalError, + "Error making backup due to #{e.message}" + end + + # Restore all dirs inside prefix. This is intended as + # restore after upgrade failure of production directories. + # + # @param backup [String] Backup path + def restore_dirs(backup) + OneCfg::LOG.unknown('Restoring from backups') + + OneCfg::Common::Backup.restore_dirs( + backup, + OneCfg::CONFIG_BACKUP_DIRS, + @prefix + ) + + OneCfg::LOG.debug('Restore successful') + rescue StandardError + msg = 'Fatal error on restore, we are very sorry! ' \ + 'You have to restore following directories ' \ + 'manually:' + + OneCfg::CONFIG_BACKUP_DIRS.each do |dir| + src = File.join(backup, dir) + dst = prefixed(dir) + + msg << "\n\t- copy #{src} into #{dst}" + end + + OneCfg::LOG.fatal(msg) + + raise + end + + end + +end diff --git a/src/onecfg/share/augeas/oned.aug b/src/onecfg/share/augeas/oned.aug deleted file mode 100644 index 1501f74806..0000000000 --- a/src/onecfg/share/augeas/oned.aug +++ /dev/null @@ -1,78 +0,0 @@ -module Oned = - autoload xfm - -(* Version: 1.4 *) - -(* Change log: *) -(* 1.4: Allow space after section *) -(* 1.3: Allow escaped quotes in values *) -(* 1.2: Include /etc/one/monitord.conf *) - -(* primitives *) -let sep = del /[ \t]*=[ \t]*/ " = " -let eol = del /\n/ "\n" -let opt_space = del /[ \t]*/ "" -let opt_space_nl = del /[ \t\n]*/ "\n" -let opt_nl_indent = del /[ \t\n]*/ "\n " -let comma = del /,/ "," -let left_br = del /\[/ "[" -let right_br = del /\]/ "]" - -(* Regexes *) -(* Match everyhting within quotes, allow escape quote *) -let re_quoted_str = /"(\\\\[\\\\"]|[^\\\\"])*"/ - -(* Match everything except spaces, quote("), l-bracket([) and num-sign(#) *) -let re_value_str = /[^ \t\n"\[#]+/ - -(* Match everything except spaces, quote("), num-sign(#) and comma(,) *) -let re_section_value_str = /[^ \t\n"#,]+/ - -(* Store either after-value comment or full-line comment *) -let comment = [ label "#comment" . store /#[^\n]*/ ] -let comment_eol = comment . eol - - -(* Simple words *) -let name = key /[A-Za-z_0-9]+/ -let re_simple_value = re_quoted_str | re_value_str - - -(* Top level entry like `PORT = 2633` *) -let top_level_entry = name . sep . store re_simple_value -let top_level_line = opt_space - . [ top_level_entry . opt_space . (comment)? ] - . eol - - -(* Section lens for section like `LOG = [ ... ]` *) -let section_value = re_quoted_str | re_section_value_str -let section_entry = [ name . sep . store section_value ] -let section_entry_list = - ( section_entry . opt_space . comma . opt_nl_indent - | comment_eol . opt_space )* - . section_entry . opt_space_nl - . ( comment_eol )* - -let section = opt_space - . [ name . sep - . left_br - . opt_nl_indent - . section_entry_list - . right_br ] - . opt_space - . eol - -let empty_line = [ del /[ \t]*\n/ "\n" ] - -(* Main lens *) -let lns = ( top_level_line | comment_eol | section | empty_line )* - - -(* Variable: filter *) -let filter = incl "/etc/one/oned.conf" - . incl "/etc/one/sched.conf" - . incl "/etc/one/monitord.conf" - . incl "/etc/one/vmm_exec/vmm_exec_kvm.conf" - -let xfm = transform lns filter diff --git a/src/onecfg/share/augeas/test_oned.aug b/src/onecfg/share/augeas/test_oned.aug deleted file mode 100644 index d635e91f39..0000000000 --- a/src/onecfg/share/augeas/test_oned.aug +++ /dev/null @@ -1,184 +0,0 @@ -module Test_oned = - - test Oned.lns get -"ENTRY = 123 -" =? - - test Oned.lns get -"ENTRY = \"MANAGE ABC\" -" =? - - test Oned.lns get -"TM_MAD_CONF = [NAME=123] -" =? - - test Oned.lns get " -A = [ NAME=123 ] -" =? - - test Oned.lns get -"A = [ -NAME=123 -] -" = ? - - test Oned.lns get -"A = [ -NAME=123, NAME2=2 -] -" = ? - - test Oned.lns get - -"#abc -LOG = [ - SYSTEM = \"file\", - DEBUG_LEVEL = 3 -] -" =? - - test Oned.lns get -"A=1 -A=1 -B=2 # comment -# abc -# - - C=[ - A=\"B\", - A=\"B\",#abc - # abc - X=\"Y\", - A=123 -] -" =? - - test Oned.lns get -"C=[ - A=123, #abc - B=223# abc -] -" -=? - test Oned.lns get -"TM_MAD = [ - EXECUTABLE = \"one_tm\", - ARGUMENTS = \"-t 15 -d dummy,lvm,shared,fs_lvm,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt\" -] -INHERIT_DATASTORE_ATTR = \"CEPH_HOST\" -" -=? - -test Oned.lns get -"LOG = [ - SYSTEM = \"file\", - DEBUG_LEVEL = 3 -] - -MONITORING_INTERVAL_HOST = 180 -MONITORING_INTERVAL_VM = 180 -MONITORING_INTERVAL_DATASTORE = 300 -MONITORING_INTERVAL_MARKET = 600 -MONITORING_THREADS = 50 - -SCRIPTS_REMOTE_DIR=/var/tmp/one -PORT = 2633 -LISTEN_ADDRESS = \"0.0.0.0\" -DB = [ BACKEND = \"sqlite\" ] - -VNC_PORTS = [ - START = 5900 -] - -FEDERATION = [ - MODE = \"STANDALONE\", - ZONE_ID = 0, - SERVER_ID = -1, - MASTER_ONED = \"\" -] - -RAFT = [ - LIMIT_PURGE = 100000, - LOG_RETENTION = 500000, - LOG_PURGE_TIMEOUT = 600, - ELECTION_TIMEOUT_MS = 2500, - BROADCAST_TIMEOUT_MS = 500, - XMLRPC_TIMEOUT_MS = 450 -] - -DEFAULT_COST = [ - CPU_COST = 0, - MEMORY_COST = 0, - DISK_COST = 0 -] - -NETWORK_SIZE = 254 - -MAC_PREFIX = \"02:00\" - -VLAN_IDS = [ - START = \"2\", - RESERVED = \"0, 1, 4095\" -] - -VXLAN_IDS = [ - START = \"2\" -] - -DATASTORE_CAPACITY_CHECK = \"yes\" - -DEFAULT_DEVICE_PREFIX = \"hd\" -DEFAULT_CDROM_DEVICE_PREFIX = \"hd\" - -DEFAULT_IMAGE_TYPE = \"OS\" -IM_MAD = [ - NAME = \"collectd\", - EXECUTABLE = \"collectd\", - ARGUMENTS = \"-p 4124 -f 5 -t 50 -i 60\" ] - -IM_MAD = [ - NAME = \"kvm\", - SUNSTONE_NAME = \"KVM\", - EXECUTABLE = \"one_im_ssh\", - ARGUMENTS = \"-r 3 -t 15 -w 90 kvm\" ] - -IM_MAD = [ - NAME = \"vcenter\", - SUNSTONE_NAME = \"VMWare vCenter\", - EXECUTABLE = \"one_im_sh\", - ARGUMENTS = \"-c -t 15 -r 0 vcenter\" ] - -IM_MAD = [ - NAME = \"ec2\", - SUNSTONE_NAME = \"Amazon EC2\", - EXECUTABLE = \"one_im_sh\", - ARGUMENTS = \"-c -t 1 -r 0 -w 600 ec2\" ] - -VM_MAD = [ - NAME = \"kvm\", - SUNSTONE_NAME = \"KVM\", - EXECUTABLE = \"one_vmm_exec\", - ARGUMENTS = \"-t 15 -r 0 kvm\", - DEFAULT = \"vmm_exec/vmm_exec_kvm.conf\", - TYPE = \"kvm\", - KEEP_SNAPSHOTS = \"no\", - IMPORTED_VMS_ACTIONS = \"terminate, terminate-hard, hold, release, suspend, - resume, delete, reboot, reboot-hard, resched, unresched, disk-attach, - disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete\" -] -" = ? - - - test Oned.lns get -"PASSWORD = \"open\\\"nebula\" -" =? - - test Oned.lns get -"DB = [ - PASSWORD = \"open\\\"nebula\" -] -" =? - - test Oned.lns get -" NIC = [ model=\"virtio\" ] -" =?