diff --git a/src/cli/oneprovision b/src/cli/oneprovision index c07b08d491..7801b9b1ea 100755 --- a/src/cli/oneprovision +++ b/src/cli/oneprovision @@ -47,6 +47,7 @@ PING_RETRIES_DEFAULT = 10 MAX_RETRIES_DEFAULT = 3 RUN_MODE_DEFAULT = :interactive FAIL_CHOICE_DEFAULT = :quit +DEFAULT_FAIL_MODES = [:cleanup, :retry, :skip, :quit] CLEANUP_DEFAULT = false THREADS_DEFAULT = 3 WAIT_TIMEOUT_DEFAULT = 60 @@ -76,6 +77,8 @@ CommandParser::CmdParser.new(ARGV) do # Global Options ######################################################################## + ############################ Output Modes ############################## + VERBOSE = { :name => 'verbose', :short => '-d', @@ -91,6 +94,8 @@ CommandParser::CmdParser.new(ARGV) do :format => String } + ############################# Run Modes ################################ + BATCH = { :name => 'batch', :short => '-b', @@ -99,6 +104,27 @@ CommandParser::CmdParser.new(ARGV) do :format => String } + FAIL_MODES = { + :name => 'fail_modes', + :large => '--fail-modes mode1,mode2', + :description => 'Fail modes to apply in order', + :format => Array, + :proc => lambda do |_, options| + options = options[:fail_modes].map do |v| + v = v.downcase.to_sym + + unless DEFAULT_FAIL_MODES.include?(v) + STDERR.puts "Wrong fail mode `#{v}`" + exit(-1) + end + + v + end + + [0, options] + end + } + FAIL_RETRY = { :name => 'fail_retry', :large => '--fail-retry number', @@ -124,21 +150,16 @@ CommandParser::CmdParser.new(ARGV) do :description => 'Set batch failover mode to quit (default)' } - FORCE = { - :name => 'force', - :short => '-F', - :large => '--force', - :description => 'Force configure to execute', - :format => String + + FAIL_SLEEP = { + :name => 'fail_sleep', + :large => '--fail-sleep seconds', + :description => 'Time in seconds between each fail mode is executed ' \ + 'and between each retry', + :format => Integer } - HARD = { - :name => 'hard', - :short => '-H', - :large => '--hard', - :description => 'Reset the host', - :format => String - } + ############################## Create ################################## PING_TIMEOUT = { :name => 'ping_timeout', @@ -156,40 +177,6 @@ CommandParser::CmdParser.new(ARGV) do :format => Integer } - THREADS = { - :name => 'threads', - :short => '-t threads', - :large => '--threads threads', - :description => "Set threads for create (default: #{THREADS_DEFAULT})", - :format => Integer - } - - CLEANUP = { - :name => 'cleanup', - :large => '--cleanup', - :description => 'Delete all vms and images first, ' \ - 'then delete the resources.' - } - - CLEANUP_TIMEOUT = { - :name => 'cleanup_timeout', - :large => '--cleanup-timeout timeout', - :description => 'Change the default timeout when deleting VMs/Images.' - } - - INCREMENTAL = { - :name => 'incremental', - :large => '--incremental', - :description => 'Configure just new hosts, default ' \ - 'is configure the whole provision.' - } - - DUMP = { - :name => 'dump', - :large => '--dump', - :description => 'Dump the configuration file result.' - } - SKIP_PROVISION = { :name => 'skip_provision', :large => '--skip-provision', @@ -215,6 +202,53 @@ CommandParser::CmdParser.new(ARGV) do :format => Integer } + ######################################################################## + + THREADS = { + :name => 'threads', + :short => '-t threads', + :large => '--threads threads', + :description => "Set threads for create (default: #{THREADS_DEFAULT})", + :format => Integer + } + + FORCE = { + :name => 'force', + :short => '-F', + :large => '--force', + :description => 'Force configure to execute', + :format => String + } + + HARD = { + :name => 'hard', + :short => '-H', + :large => '--hard', + :description => 'Reset the host', + :format => String + } + + CLEANUP = { + :name => 'cleanup', + :large => '--cleanup', + :description => 'Delete all vms and images first, ' \ + 'then delete the resources.' + } + + CLEANUP_TIMEOUT = { + :name => 'cleanup_timeout', + :large => '--cleanup-timeout timeout', + :description => 'Change the default timeout when deleting VMs/Images.' + } + + DUMP = { + :name => 'dump', + :large => '--dump', + :description => 'Dump the configuration file result.' + } + + ######################################################################## + MODES = CommandParser::OPTIONS - [CommandParser::VERBOSE] + [VERBOSE, DEBUG, @@ -222,7 +256,9 @@ CommandParser::CmdParser.new(ARGV) do FAIL_RETRY, FAIL_CLEANUP, FAIL_SKIP, - FAIL_QUIT] + FAIL_QUIT, + FAIL_MODES, + FAIL_SLEEP] CREATE_OPTIONS = [THREADS, MODES, diff --git a/src/oneprovision/lib/driver.rb b/src/oneprovision/lib/driver.rb index 5b75e6a978..216046b2ae 100644 --- a/src/oneprovision/lib/driver.rb +++ b/src/oneprovision/lib/driver.rb @@ -40,12 +40,15 @@ module OneProvision STDERR.puts "ERROR: #{text}\n#{e.text}" retries += 1 + choice = Mode.fail_choice + seconds = Mode.fail_sleep if retries > Mode.max_retries && Mode.mode == :batch - exit(-1) - end + retries = 0 + choice = Mode.next_fail_choice - choice = Mode.fail_choice + sleep(seconds) if seconds && choice + end if Mode.mode == :interactive begin @@ -67,11 +70,13 @@ module OneProvision end if choice == :retry + sleep(seconds) if seconds + retry elsif choice == :quit exit(-1) elsif choice == :skip - return + return :skip elsif choice == :cleanup raise OneProvisionCleanupException if cleanup diff --git a/src/oneprovision/lib/oneprovision.rb b/src/oneprovision/lib/oneprovision.rb index 3b94941f23..525f0ace7a 100644 --- a/src/oneprovision/lib/oneprovision.rb +++ b/src/oneprovision/lib/oneprovision.rb @@ -108,16 +108,28 @@ module OneProvision instance.run_mode[:mode] = RUN_MODE_DEFAULT end - if options.key? :fail_cleanup - instance.run_mode[:fail_choice] = :cleanup - elsif options.key? :fail_retry - instance.run_mode[:fail_choice] = :retry - elsif options.key? :fail_skip - instance.run_mode[:fail_choice] = :skip - elsif options.key? :fail_quit - instance.run_mode[:fail_choice] = :quit - else - instance.run_mode[:fail_choice] = FAIL_CHOICE_DEFAULT + instance.run_mode[:fail_modes] = [] + + if options.key? :fail_modes + instance.run_mode[:fail_modes] << options[:fail_modes] + end + + options.each do |key, _| + next unless key =~ /^fail_/ + + next if key =~ /sleep/ || key =~ /modes/ + + instance.run_mode[:fail_modes] << key.to_s.split('_')[1].to_sym + end + + if instance.run_mode[:fail_modes].empty? + instance.run_mode[:fail_modes] << FAIL_CHOICE_DEFAULT + end + + instance.run_mode[:fail_modes].flatten! + + if options.key? :fail_sleep + instance.run_mode[:fail_sleep] = options[:fail_sleep] end if options[:fail_retry] @@ -140,7 +152,21 @@ module OneProvision # # @return [Sym] Choice value def self.fail_choice - instance.run_mode[:fail_choice] + instance.run_mode[:fail_modes][0] + end + + # Gets time between fail modes + # + # @return [Integer] Time in seconds + def self.fail_sleep + instance.run_mode[:fail_sleep] + end + + # Gets next fail choice value + # + # @return [Sym] Choice value + def self.next_fail_choice + instance.run_mode[:fail_modes].shift end # Gets max retries value diff --git a/src/oneprovision/lib/provision.rb b/src/oneprovision/lib/provision.rb index 73df597502..4b2877ac99 100644 --- a/src/oneprovision/lib/provision.rb +++ b/src/oneprovision/lib/provision.rb @@ -199,11 +199,14 @@ module OneProvision cluster = nil cid = nil - Driver.retry_loop 'Failed to create cluster' do + rc = Driver.retry_loop 'Failed to create cluster' do cluster = create_cluster(cfg) cid = cluster.id end + # If cluster fails to create and user select skip, exit + exit if rc == :skip + Mode.new_cleanup(true) create_resources(cfg, cid)