shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-03-30 22:50:10 +03:00
Code Releases Wiki Activity

F : implement new syntax in provision tpls ()

Browse Source
This commit is contained in:
Alejandro Huertas Herrero 2020-11-12 15:32:12 +01:00 committed by GitHub
parent 9d6d90b486
commit 3da0fb41c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 749 additions and 520 deletions

1
install.sh

@ -2362,6 +2362,7 @@ ONEPROVISION_LIB_PROVISION_FILES="src/oneprovision/lib/provision/ansible.rb \
src/oneprovision/lib/provision/oneprovision.rb \
src/oneprovision/lib/provision/driver.rb \
src/oneprovision/lib/provision/provision.rb \
src/oneprovision/lib/provision/provision_config.rb \
src/oneprovision/lib/provision/provision_pool.rb \
src/oneprovision/lib/provision/resources.rb \
src/oneprovision/lib/provision/utils.rb"

5
src/cli/one_helper/oneprovision_template_helper.rb

@ -167,8 +167,9 @@ class OneProvisionTemplateHelper < OpenNebulaHelper::OneHelper
# @return [YAML] Template in YAML format
def validate(template)
begin
template = OneProvision::Utils.read_config(template)
provider = OneProvision::Provision.read_provider(template)
config = OneProvision::ProvisionConfig.new(template)
template = config.validate
provider = OneProvision::Provision.read_provider(config)
raise 'Name not found' unless template['name']
raise 'Provider not found in template' unless provider

7
src/cli/oneprovision

@ -144,9 +144,12 @@ CommandParser::CmdParser.new(ARGV) do
validate_desc,
[:config_file],
:options => OneProvisionHelper::DUMP do
dump = options.key? :dump
config = OneProvision::ProvisionConfig.new(args[0])
config = config.validate
OneProvision::Utils.validate_configuration(args[0], dump)
puts config.to_yaml if (options.key? :dump)
0
end
###

10
src/oneprovision/lib/provision/ansible.rb

@ -104,7 +104,7 @@ module OneProvision
)
hosts.each do |h|
host = Host.new
host = Resource.object('hosts')
host.info(h['id'])
host.one.enable
@ -252,7 +252,7 @@ module OneProvision
c = "[nodes]\n"
hosts.each do |h|
host = Host.new
host = Resource.object('hosts')
host.info(h['id'])
c << "#{host.one['NAME']}\n"
@ -263,7 +263,7 @@ module OneProvision
c << "[targets]\n"
hosts.each do |h|
host = Host.new
host = Resource.object('hosts')
host.info(h['id'])
conn = get_host_template_conn(host.one)
@ -281,7 +281,7 @@ module OneProvision
Dir.mkdir("#{ansible_dir}/host_vars")
hosts.each do |h|
host = Host.new
host = Resource.object('hosts')
host.info(h['id'])
var = host.one['TEMPLATE/PROVISION_CONFIGURATION_BASE64']
@ -292,7 +292,7 @@ module OneProvision
Driver.write_file_log(fname, c)
end
host = Host.new
host = Resource.object('hosts')
host.info(hosts[0]['id'])
if host.one['TEMPLATE/ANSIBLE_PLAYBOOK']

1
src/oneprovision/lib/provision/oneprovision.rb

@ -17,6 +17,7 @@
require 'provision/ansible'
require 'provision/driver'
require 'provision/provision'
require 'provision/provision_config'
require 'provision/provision_pool'
require 'provision/resources'
require 'provision/utils'

199
src/oneprovision/lib/provision/provision.rb

@ -19,6 +19,9 @@ module OneProvision
# Provision class as wrapper of DocumentJSON
class Provision < ProvisionElement
# @idx [Integer] Index used when creating multiple objects
attr_reader :idx
DOCUMENT_TYPE = 103
STATE = {
@ -97,6 +100,11 @@ module OneProvision
@body['provision']['infrastructure']
end
# Get cluster information
def cluster
infrastructure_objects['clusters'][0]
end
# Returns provision hosts
def hosts
infrastructure_objects['hosts']
@ -114,9 +122,7 @@ module OneProvision
# Returns provision provider
def provider
if @body['provider'] == 'dummy'
return { 'NAME' => 'dummy' }
end
return { 'NAME' => 'dummy' } if @body['provider'] == 'dummy'
Provider.by_name(@client, @body['provider'])
end
@ -191,8 +197,15 @@ module OneProvision
def deploy(config, cleanup, timeout, skip, provider)
Ansible.check_ansible_version if skip == :none
cfg = ProvisionConfig.new(config)
cfg.validate
begin
cfg = Utils.read_config(config)
@idx = nil
# Create configuration object
cfg = ProvisionConfig.new(config)
cfg.load
# read provider information
unless provider
@ -206,6 +219,12 @@ module OneProvision
return OpenNebula::Error.new('No provider found')
end
@provider = provider
allocate(cfg, provider)
info(true)
# Respect user information, only add provider info if
# the user hasn't specified any
cfg['defaults'] = {} unless cfg['defaults']
@ -219,21 +238,21 @@ module OneProvision
conn = provider.connection
if uid == provider['UID']
cfg['defaults']['provision'] = conn.merge(
cfg['defaults']['provision']
cfg['defaults/provision'] = conn.merge(
cfg['defaults/provision']
)
OneProvisionLogger.debug('Merging provider connection')
else
cfg['defaults']['provision'] = conn
cfg['defaults/provision'] = conn if conn
OneProvisionLogger.debug('Using provider connection')
end
# read provision file
cfg = Utils.create_config(cfg)
cfg.parse
# @name is used for ERB evaluation
# @name is used for template evaluation
@name = cfg['name']
OneProvisionLogger.info('Creating provision objects')
@ -245,16 +264,12 @@ module OneProvision
# If cluster fails to create and user select skip, exit
exit if rc == :skip
create_infra_resources(cfg, cfg['cluster']['id'])
create_hosts(cfg, cfg['cluster']['id'])
allocate(cfg, provider)
create_infra_resources(cfg)
create_hosts(cfg)
Mode.new_cleanup(true)
self.info(true)
# @id is used for ERB evaluation
# @id is used for template evaluation
@id = self['ID']
if skip != :all && hosts && !hosts.empty?
@ -281,6 +296,8 @@ module OneProvision
@body['tf'] = {}
@body['tf']['state'] = state
@body['tf']['conf'] = conf
update
end
if skip == :none
@ -368,17 +385,19 @@ module OneProvision
OneProvisionLogger.info("Deleting provision #{self['ID']}")
if !hosts.empty? && tf_state && tf_conf
if hosts && !hosts.empty? && tf_state && tf_conf
Driver.tf_action(self, 'destroy', tf)
end
Driver.retry_loop 'Failed to delete hosts' do
hosts.each do |host|
id = host['id']
host = Host.new(provider['NAME'])
if hosts
Driver.retry_loop 'Failed to delete hosts' do
hosts.each do |host|
id = host['id']
host = Host.new(provider)
host.info(id)
host.delete
host.info(id)
host.delete
end
end
end
@ -445,11 +464,6 @@ module OneProvision
update
end
# Returns the binding of the class
def _binding
binding
end
# Reads provider name from template
#
# @param template [Hash] Provision information
@ -460,8 +474,8 @@ module OneProvision
# Provider can be set in provision defaults or in hosts
# it's the same for all hosts, taking the first one is enough
if template['defaults'] && template['defaults']['provision']
provider = template['defaults']['provision']['provider']
if template['defaults'] && template['defaults/provision']
provider = template['defaults/provision/provider']
end
if !provider && template['hosts'][0]['provision']
@ -497,20 +511,6 @@ module OneProvision
document['provision'] = {}
document['provision']['infrastructure'] = {}
FULL_CLUSTER.each do |r|
next unless template[r]
document['provision']['infrastructure'][r] = []
template[r].each do |o|
obj = {}
obj['name'] = o['name']
obj['id'] = o['id']
document['provision']['infrastructure'][r] << obj
end
end
# Resources are allocated later
document['provision']['resource'] = {}
@ -523,44 +523,46 @@ module OneProvision
#
# @return [OpenNebula::Cluster]
def create_cluster(cfg)
msg = "Creating OpenNebula cluster: #{cfg['cluster']['name']}"
msg = "Creating OpenNebula cluster: #{cfg['cluster/name']}"
OneProvisionLogger.debug(msg)
cluster = Cluster.new
id = cluster.create(cfg['cluster'])
obj = Cluster.new(nil, cfg['cluster'])
# Update cluster information in template
cfg['cluster']['id'] = id
obj.evaluate_rules(self)
cfg['clusters'] = []
cfg['clusters'] << { 'name' => cfg['cluster']['name'], 'id' => id }
id = obj.create
infrastructure_objects['clusters'] = []
infrastructure_objects['clusters'] << { 'id' => id,
'name' => obj.one['NAME'] }
OneProvisionLogger.debug("Cluster created with ID: #{id}")
update
end
# Creates provision infrastructure resources
#
# @param cfg [Hash] Provision information
# @param resources [Array] Resource names
# @param cid [Integer] Cluster ID
def create_resources(cfg, resources, cid = nil)
def create_resources(cfg, resources)
cid = Integer(cluster['id'])
resources.each do |r|
next if cfg[r].nil?
cfg[r].each do |x|
Driver.retry_loop 'Failed to create some resources' do
obj = Resource.object(r)
obj = Resource.object(r, nil, x)
next if obj.nil?
x = Utils.evaluate_erb(self, x)
OneProvisionLogger.debug(
"Creating #{r[0..-2]} #{x['name']}"
)
yield(r, obj, x, cid)
yield(r, obj, cid)
obj.template_chown(x)
obj.template_chmod(x)
@ -573,15 +575,17 @@ module OneProvision
# Creates provision infrastructure resources
#
# @param cfg [Hash] Provision information
# @param cid [Integer] Cluster ID
def create_infra_resources(cfg, cid)
create_resources(cfg, INFRASTRUCTURE_RESOURCES, cid) do |_,
obj,
x,
c|
x['id'] = obj.create(x, c)
x['name'] = obj.one['NAME']
# @param cfg [Hash] Provision information
def create_infra_resources(cfg)
create_resources(cfg, INFRASTRUCTURE_RESOURCES) do |r, obj, c|
obj.evaluate_rules(self)
infrastructure_objects[r] = [] unless infrastructure_objects[r]
id = obj.create(c)
infrastructure_objects[r] << { 'id' => id,
'name' => obj.one['NAME'] }
end
end
@ -589,8 +593,10 @@ module OneProvision
#
# @param cfg [Hash] Provision information
def create_virtual_resources(cfg)
create_resources(cfg, RESOURCES) do |r, obj, x, _|
ret = obj.create(x)
create_resources(cfg, RESOURCES) do |r, obj, _|
obj.evaluate_rules(self)
ret = obj.create
resource_objects[r] = [] unless resource_objects[r]
if ret.is_a? Array
@ -614,48 +620,45 @@ module OneProvision
# Creates provision hosts
#
# @param cfg [Hash] Provision information
# @param cid [Integer] Cluster ID
def create_hosts(cfg, cid)
# @param cfg [Hash] Provision information
def create_hosts(cfg)
return unless cfg['hosts']
hosts = []
infrastructure_objects['hosts'] = []
cid = Integer(cluster['id'])
cfg['hosts'].each do |h|
h['count'].nil? ? count = 1 : count = Integer(h['count'])
# Store original host template
h_bck = Marshal.load(Marshal.dump(h))
count.times.each do |idx|
@idx = idx
Driver.retry_loop 'Failed to create some host' do
# Update hostname to avoid multiple same names
hostname = h['provision']['hostname'] if h['provision']
if hostname && count > 1
h['provision']['hostname'] = "#{hostname}_#{idx}"
end
erb = Utils.evaluate_erb(self, h)
dfile = Utils.create_deployment_file(erb)
playbooks = cfg['playbook']
playbooks = playbooks.join(',') if playbooks.is_a? Array
host = Host.new
host = host.create(dfile.to_xml, cid, playbooks)
h = Marshal.load(Marshal.dump(h_bck))
host = Resource.object('hosts', @provider, h)
if count > 1 && idx > 0
hosts << { 'id' => Integer(host['ID']),
'name' => host['NAME'] }
else
h['id'] = Integer(host['ID'])
h['name'] = host['NAME']
end
host.evaluate_rules(self)
dfile = host.create_deployment_file
host = host.create(dfile.to_xml, cid, playbooks)
obj = { 'id' => Integer(host['ID']),
'name' => host['NAME'] }
infrastructure_objects['hosts'] << obj
host.offline
update
end
end
end
cfg['hosts'] += hosts
end
# Updates provision hosts with new name
@ -664,7 +667,7 @@ module OneProvision
# @param ids [Array] IDs for each host
def update_hosts(ips, ids)
hosts.each do |h|
host = Host.new(provider)
host = Resource.object('hosts', provider)
host.info(h['id'])
name = ips.shift
@ -688,7 +691,7 @@ module OneProvision
return unless hosts
hosts.each do |h|
host = Host.new
host = Resource.object('hosts')
Utils.exception(host.info(h['id']))
@ -705,7 +708,7 @@ module OneProvision
return unless datastores
datastores.each do |d|
datastore = Datastore.new
datastore = Resource.object('datastores')
Utils.exception(datastore.info(d['id']))
@ -735,7 +738,7 @@ module OneProvision
d_hosts = []
hosts.each do |h|
host = Host.new
host = Resource.object('hosts')
Utils.exception(host.info(h['id']))
@ -766,7 +769,7 @@ module OneProvision
d_datastores = []
datastores.each do |d|
datastore = Datastore.new
datastore = Resource.object('datastores')
Utils.exception(datastore.info(d['id']))

456
src/oneprovision/lib/provision/provision_config.rb Normal file

@ -0,0 +1,456 @@
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
module OneProvision
# Provision configuration class
#
# Class to manage configuration file used to deploy a provision
#
# STEPS
############################################################################
#
# 1. Load configuration file
# 2. Parse configuration to add provision defaults sections
# 3. Check that objects have specific sections
# 4. Check evaluation rules they should follow:
#
# ${object.name.attr}
#
# where:
#
# - object: valid values Resource::EVAL_KEYS
# - name: any alphanumeric string
# - attr: object attribute located at first level
class ProvisionConfig
# Class constructor
#
# @param template [String/Hash]
# String -> Path to configuration YAML file
# Hash -> configuration file already loaded
def initialize(template)
case template
when Hash
@config = template
else
@config_file = template
end
end
# Loads configuration file
def load
@config = partial_load(@config_file)
end
# Parses configuration hash to add defaults into each section
def parse
begin
defaults = @config['defaults']
################################################################
# Cluster
################################################################
if @config['cluster'].nil?
@config['cluster'] = { 'name' => @config['name'] }
end
@config['cluster']['provision'] ||= {}
if defaults && defaults.key?('provision')
@config['cluster']['provision'].merge!(
defaults['provision']
)
end
################################################################
# Hosts
################################################################
if @config['hosts']
sections = %w[connection provision configuration]
@config['hosts'].map! do |host|
sections.each do |section|
data = CONFIG_DEFAULTS[section] || {}
if @config['defaults']
defaults = @config['defaults'][section]
end
h_sec = host[section]
# merge defaults with globals
# and device specific params
data.merge!(defaults) unless defaults.nil?
data.merge!(h_sec) unless h_sec.nil?
host[section] = data
end
host
end
end
################################################################
# Datastores & Networks
################################################################
%w[datastores networks].each do |r|
next unless @config[r]
@config[r].map! do |x|
x['provision'] ||= {}
if defaults && defaults.key?('provision')
x['provision'].merge!(defaults['provision'])
end
x
end
end
rescue StandardError => e
Utils.fail("Failed to read configuration: #{e}")
end
end
# Checks configuration file for some specifics attributes
#
# - Each host should have im_mad, vm_mad and hostname
# - Each datastore should have tm_mad and ds_mad
# - Each network should have vn_mad
def check
if @config['name'].nil?
Utils.fail('in your configuration file: no name given')
end
if @config['hosts']
@config['hosts'].each_with_index do |h, i|
if h['im_mad'].nil?
Utils.fail("in configuration file: no im_mad #{i + 1}")
end
if h['vm_mad'].nil?
Utils.fail("in configuration file: no vm_mad #{i + 1}")
end
next unless h['provision']['hostname'].nil?
Utils.fail("in configuration file: no hostname #{i + 1}")
end
end
if @config['datastores']
@config['datastores'].each_with_index do |d, i|
if d['tm_mad'].nil?
Utils.fail("in configuration file: no tm_mad #{i + 1}")
end
next if d['type']
next if d['ds_mad']
Utils.fail("in configuration file: no ds_mad #{i + 1}")
end
end
return unless @config['networks']
@config['networks'].each_with_index do |n, i|
next unless n['vn_mad'].nil?
Utils.fail("in configuration file: no vn_mad #{i + 1}")
end
end
# Checks evaluation rules
def check_rules
iterate(@config) do |value|
rc = check_rule(value)
next if rc[0]
Utils.fail("expression #{value}: #{rc[1]}")
end
end
# Evaluates all rules
#
# @param provision [Provision] Provision object
def eval_rules(provision)
iterate(@config, true) do |value|
matches = value.to_s.scan(/\$\{(.*?)\}/).flatten
unless matches.empty?
matches.each do |match|
# match[0]: object
# match[1]: name
# match[2]: attribute
match = match.split('.')
if match.size == 1
value.gsub!('${provision}', provision.name.to_s)
value.gsub!('${provision_id}', provision.id.to_s)
if provision.idx
value.gsub!('${index}', provision.idx.to_s)
end
else
objects = provision.info_objects("#{match[0]}s")
object = objects.find do |obj|
obj['NAME'] == match[1]
end
object = object.to_hash
object = object[object.keys[0]]
value.gsub!("${#{match.join('.')}}",
object[match[2].upcase])
end
end
end
value
end
end
# Validates the configuration file
#
# Check if file can be loaded
# Parse it to merge different sections
# Check specific attributes
# Check all evaluation rules
def validate
self.load
@config.delete_if {|_k, v| v.nil? }
parse
check
check_rules
@config
end
########################################################################
# Helper functions
########################################################################
# Modifies configuration hash
#
# @param path [String] Path separated with /
# @param value [String] Value to set
def []=(path, value)
path = path.split('/')
hash = dsearch(path[0..-2])
hash[path[-1]] = value
end
# Gets value from configuration hash
#
# @param path [String] Path separated with /
#
# @return [String] Value in the path
def [](path)
dsearch(path.split('/'))
end
private
# Reads configuration content
#
# @param name [String] Path to the configuration file
#
# @return [Hash] Configuration content
def partial_load(name)
begin
yaml = YAML.load_file(name)
rescue StandardError => e
Utils.fail("Failed to read template: #{e}")
end
if yaml['extends']
yaml['extends'] = [yaml['extends']].flatten
yaml['extends'].reverse.each do |f|
base = partial_load(f)
yaml.delete('extends')
base['defaults'] ||= {}
yaml['defaults'] ||= {}
if base['playbook']
playbooks = []
playbooks << base['playbook']
playbooks << yaml['playbook'] if yaml['playbook']
playbooks.flatten!
yaml['playbook'] = playbooks
base.delete('playbook')
end
if yaml['playbook']
yaml['playbook'] = [yaml['playbook']]
yaml['playbook'].flatten!
end
# replace scalars or append array from child YAML
yaml.each do |key, value|
next if key == 'defaults'
if (value.is_a? Array) && (base[key].is_a? Array)
base[key].concat(value)
else
base[key] = value
end
end
# merge each defaults section separately
%w[connection provision configuration].each do |section|
base['defaults'][section] ||= {}
yaml['defaults'][section] ||= {}
defaults = yaml['defaults'][section]
base['defaults'][section].merge!(defaults)
end
yaml = base
end
end
yaml
end
# Iterates over objects
#
# @param objects [Object] Objects to iterate over
# @param ev [Boolean] True to assign value
def iterate(objects, ev = false, &block)
if objects.is_a? Hash
objects.each do |key, value|
case value
when Hash
iterate(value, ev, &block)
when Array
if ev
value.map! {|el| iterate(el, ev, &block) }
else
value.each {|el| iterate(el, ev, &block) }
end
else
ret = yield(value)
objects[key] = ret if ev
end
end
else
yield(objects)
end
end
# Check that evaluation rule fixes the format
#
# @param rule [String] Rule to check
#
# @return [Boolean, String]
# True, '' if rule is correct
# False, 'Error message' otherwhise
def check_rule(rule)
rule = rule.to_s
matches = rule.scan(/\$\{(.*?)\}/).flatten
# Skip the rule as it does not fit in ${} pattern
return [true, ''] if matches.empty?
matches.each do |match|
match = match.split('.')
# Special evaluation for keys provision, provison_id and idx
if match.size == 1 && !Resource::S_EVAL_KEYS.include?(match[0])
return [false, "key #{match[0]} invalid"]
end
next if match.size == 1
# Rules can only access to first level they must be
# resource.name.attr
return [false, 'there are no 3 elements'] if match.size != 3
# Only a group of key words is available
unless Resource::EVAL_KEYS.include?(match[0])
return [false, "key #{match[0]} is not valid"]
end
# Every part of the rule can only have numbers, letters, _ and -
rc = match.each do |m|
next if m[/[a-zA-Z0-9_-]+/] == m
break [false, "#{m} has invalid characters"]
end
return rc if rc[0] == false
# Check that referenced names can be found in @config
elements = @config[match[0]] || @config["#{match[0]}s"] || []
elements = [elements].flatten
# Add market apps so user can refer to them
if @config['marketplaceapps']
elements += @config['marketplaceapps']
end
unless elements.find {|v| v['name'] == match[1] }
return [false, "#{match[0]} #{match[1]} not found"]
end
end
[true, '']
end
# Search inside path
#
# @param path [String] Path to search on
def dsearch(path)
hash = @config
path.delete_if {|s| s.nil? || s.empty? }
path.each do |p|
if hash.is_a? Hash
if hash[p]
hash = hash[p]
else
hash = nil
break
end
else
hash = nil
break
end
end
hash
end
end
end

15
src/oneprovision/lib/provision/resources/physical/cluster.rb

@ -23,9 +23,10 @@ module OneProvision
# Class constructor
#
# @param provider [provider] Cluster provider
def initialize(provider = nil)
super()
# @param provider [Provider] Cluster provider
# @param p_template [Hash] Resource information in hash form
def initialize(provider, p_template = nil)
super(p_template)
@pool = OpenNebula::ClusterPool.new(@client)
@type = 'cluster'
@ -34,15 +35,13 @@ module OneProvision
# Creates a new cluster in OpenNebula
#
# @param template [Hash] Cluster template information
#
# @return [Integer] Resource ID
def create(template)
def create
new_object
rc = @one.allocate(template['name'])
rc = @one.allocate(@p_template['name'])
Utils.exception(rc)
rc = @one.update(Utils.template_like_str(template), true)
rc = @one.update(Utils.template_like_str(@p_template), true)
Utils.exception(rc)
rc = @one.info
Utils.exception(rc)

5
src/oneprovision/lib/provision/resources/physical/datastore.rb

@ -23,8 +23,9 @@ module OneProvision
# Class constructor
#
# @param provider [Provider] Datastore provider
def initialize(provider = nil)
# @param provider [Provider] Datastore provider
# @param p_template [Hash] Resource information in hash form
def initialize(provider, p_template = nil)
super
@pool = OpenNebula::DatastorePool.new(@client)

66
src/oneprovision/lib/provision/resources/physical/host.rb

@ -28,15 +28,75 @@ module OneProvision
# Class constructor
#
# @param provider [Provider] Host provider
def initialize(provider = nil)
super()
# @param provider [Provider] Host provider
# @param p_template [Hash] Resource information in hash form
def initialize(provider, p_template = nil)
super(p_template)
@pool = OpenNebula::HostPool.new(@client)
@type = 'host'
@provider = provider
end
# Creates host deployment file
#
# @return [Nokogiri::XML] XML with the host information
def create_deployment_file
ssh_key = Utils.try_read_file(
@p_template['connection']['public_key']
)
config = Base64.strict_encode64(
@p_template['configuration'].to_yaml
)
reject = %w[im_mad vm_mad provision connection configuration]
Nokogiri::XML::Builder.new do |xml|
xml.HOST do
xml.NAME "provision-#{SecureRandom.hex(24)}"
xml.TEMPLATE do
xml.IM_MAD @p_template['im_mad']
xml.VM_MAD @p_template['vm_mad']
xml.PROVISION do
@p_template['provision'].each do |key, value|
next if key == 'provider'
xml.send(key.upcase, value)
end
xml.send('PROVIDER', @provider['NAME'])
end
if @p_template['configuration']
xml.PROVISION_CONFIGURATION_BASE64 config
end
if @p_template['connection']
xml.PROVISION_CONNECTION do
@p_template['connection'].each do |key, value|
xml.send(key.upcase, value)
end
end
end
if @p_template['connection']
xml.CONTEXT do
if @p_template['connection']['public_key']
xml.SSH_PUBLIC_KEY ssh_key
end
end
end
@p_template.each do |key, value|
next if reject.include?(key)
xml.send(key.upcase, value)
end
end
end
end.doc.root
end
# Checks if there are Running VMs on the HOST
def running_vms?
Integer(@one['HOST_SHARE/RUNNING_VMS']) > 0

5
src/oneprovision/lib/provision/resources/physical/network.rb

@ -23,8 +23,9 @@ module OneProvision
# Class constructor
#
# @param provider [provider] Network provider
def initialize(provider = nil)
# @param provider [Provider] Network provider
# @param p_template [Hash] Resource information in hash form
def initialize(provider, p_template = nil)
super
@pool = OpenNebula::VirtualNetworkPool.new(@client)

12
src/oneprovision/lib/provision/resources/physical/physical_resource.rb

@ -23,24 +23,24 @@ module OneProvision
# Class constructor
#
# @param provider [Provider] Resource provider
def initialize(provider)
super()
# @param provider [Provider] Resource provider
# @param p_template [Hash] Resource information in hash form
def initialize(provider, p_template)
super(p_template)
@provider = provider
end
# Creates the object in OpenNebula
#
# @param template [Hash] Object attributes
# @param cluster_id [Integer] Cluster ID
#
# @return [Integer] Resource ID
def create(template, cluster_id)
def create(cluster_id)
# create ONE object
new_object
rc = @one.allocate(format_template(template), cluster_id)
rc = @one.allocate(format_template(@p_template), cluster_id)
Utils.exception(rc)
rc = @one.info
Utils.exception(rc)

64
src/oneprovision/lib/provision/resources/resource.rb

@ -22,13 +22,46 @@ module OneProvision
# Keys to remove from template
REJECT_KEYS = %w[meta]
# Valid keys in template evaluation
EVAL_KEYS = %w[cluster
datastore
host
image
network
template
vntemplate
marketplaceapp]
S_EVAL_KEYS = %w[index
provision
provision_id]
# @one ONE object
# @pool ONE pool
attr_reader :one, :pool
# Class constructor
def initialize
@client = OpenNebula::Client.new
#
# @param p_template [Hash] Resource information in hash form
def initialize(p_template)
@client = OpenNebula::Client.new
@p_template = p_template
end
# Evaluate provision template rules that reference other objects
# in the provision.
#
# Rules with format ${object.name.attr} will be transformed by the
# real value.
#
# All the references to provision or provision_id will be changed by
# the provision name and the provision ID.
#
# @param provision [Provision] Provision information
def evaluate_rules(provision)
config = ProvisionConfig.new(@p_template)
config.eval_rules(provision)
end
# Create operation is implemented in childs
@ -38,30 +71,31 @@ module OneProvision
# Factory to return new object
#
# @param type [String] Object type
# @param provider [Provider] Provider to execute remote operations
def self.object(type, provider = nil)
# @param type [String] Object type
# @param provider [Provider] Provider to execute remote operations
# @param p_template [Hash] Resource information in hash form
def self.object(type, provider = nil, p_template = nil)
type = type.downcase[0..-2].to_sym
case type
when :cluster
Cluster.new(provider)
Cluster.new(provider, p_template)
when :datastore
Datastore.new(provider)
Datastore.new(provider, p_template)
when :host
Host.new(provider)
Host.new(provider, p_template)
when :image
Image.new
Image.new(p_template)
when :network
Network.new(provider)
Network.new(provider, p_template)
when :template
Template.new
Template.new(p_template)
when :vntemplate
VnTemplate.new
VnTemplate.new(p_template)
when :flowtemplate
FlowTemplate.new
FlowTemplate.new(p_template)
when :marketplaceapp
MarketPlaceApp.new
MarketPlaceApp.new(p_template)
else
nil
end
@ -186,7 +220,7 @@ module OneProvision
"Chown #{@type} #{@one.id} #{user}:#{group}"
)
rc = @one.chown(user, group)
rc = @one.chown(Integer(user), Integer(group))
return unless OpenNebula.is_error?(rc)

4
src/oneprovision/lib/provision/resources/virtual/flowtemplate.rb

@ -25,7 +25,9 @@ module OneProvision
class FlowTemplate < VirtualResource
# Class constructor
def initialize
#
# @param p_template [Hash] Resource information in hash form
def initialize(p_template)
super
@pool = OpenNebula::ServiceTemplatePool.new(@client)

26
src/oneprovision/lib/provision/resources/virtual/image.rb

@ -22,7 +22,9 @@ module OneProvision
class Image < VirtualSyncResource
# Class constructor
def initialize
#
# @param p_template [Hash] Resource information in hash form
def initialize(p_template = nil)
super
@pool = OpenNebula::ImagePool.new(@client)
@ -31,11 +33,9 @@ module OneProvision
# Creates a new object in OpenNebula
#
# @param template [Hash] Object attributes
#
# @return [Integer] Resource ID
def create(template)
meta = template['meta']
def create
meta = @p_template['meta']
wait, timeout = OneProvision::ObjectOptions.get_wait(meta)
info = { 'wait' => wait,
@ -43,13 +43,13 @@ module OneProvision
check_wait(wait)
add_provision_info(template, info)
add_provision_info(@p_template, info)
# create ONE object
new_object
rc = @one.allocate(format_template(template),
Integer(template['ds_id']))
rc = @one.allocate(format_template(@p_template),
Integer(@p_template['ds_id']))
Utils.exception(rc)
rc = @one.info
Utils.exception(rc)
@ -60,7 +60,7 @@ module OneProvision
return Integer(@one.id) unless wait
ready?(template)
ready?
end
# Info an specific object
@ -80,12 +80,10 @@ module OneProvision
# Wait until the image is ready, retry if fail
#
# @param template [Hash] Object attributes
#
# @return [Integer] Resource ID
def ready?(template)
def ready?
Driver.retry_loop 'Fail to create image' do
wait_state('READY', template['timeout'])
wait_state('READY', @p_template['timeout'])
# check state after existing wait loop
@one.info
@ -93,7 +91,7 @@ module OneProvision
case @one.state_str
when 'LOCKED'
# if locked, keep waiting
ready?(template)
ready?
when 'ERROR'
# if error, delete the image and try to create it again
raise OneProvisionLoopException

36
src/oneprovision/lib/provision/resources/virtual/marketplaceapp.rb

@ -22,7 +22,9 @@ module OneProvision
class MarketPlaceApp < VirtualSyncResource
# Class constructor
def initialize
#
# @param p_template [Hash] Resource information in hash form
def initialize(p_template)
super
@type = 'marketplaceapp'
@ -30,13 +32,11 @@ module OneProvision
# Creates a new object in OpenNebula
#
# @param template [String] Object template
#
# @return [Integer] Resource ID
def create(template)
meta = template['meta']
def create
meta = @p_template['meta']
app_id = template['appid'] || name_to_id(template['appname'])
app_id = @p_template['appid'] || name_to_id(@p_template['appname'])
wait, timeout = OneProvision::ObjectOptions.get_wait(meta)
check_wait(wait)
@ -56,15 +56,15 @@ module OneProvision
Utils.exception(rc)
app.extend(MarketPlaceAppExt)
url_args = "tag=#{template['tag']}" if template['tag']
url_args = "tag=#{@p_template['tag']}" if @p_template['tag']
rc = app.info
Utils.exception(rc)
rc = app.export(
:dsid => Integer(template['dsid']),
:name => template['name'],
:vmtemplate_name => template['vmname'],
:dsid => Integer(@p_template['dsid']),
:name => @p_template['name'],
:vmtemplate_name => @p_template['vmname'],
:url_args => url_args
)
Utils.exception(rc[:image].first) if rc[:image]
@ -94,17 +94,19 @@ module OneProvision
'wait_timeout' => timeout })
# Change permissions and ownership
@image.template_chown(template)
@image.template_chmod(template)
@template.template_chown(template)
@template.template_chmod(template)
@image.template_chown(@p_template)
@image.template_chmod(@p_template)
@template.template_chown(@p_template)
@template.template_chmod(@p_template)
return [image_id, template_id] unless wait
ret = [{ 'id' => image_id, 'name' => @image.one['NAME'] },
{ 'id' => template_id, 'name' => @template.one['NAME'] }]
return ret unless wait
@image.wait_state('READY', timeout)
[{ 'id' => image_id, 'name' => @image.one['NAME'] },
{ 'id' => template_id, 'name' => @template.one['NAME'] }]
ret
end
########################################################################

4
src/oneprovision/lib/provision/resources/virtual/template.rb

@ -22,7 +22,9 @@ module OneProvision
class Template < VirtualResource
# Class constructor
def initialize
#
# @param p_template [Hash] Resource information in hash form
def initialize(p_template = nil)
super
@pool = OpenNebula::TemplatePool.new(@client)

6
src/oneprovision/lib/provision/resources/virtual/virtual_resource.rb

@ -23,14 +23,12 @@ module OneProvision
# Creates a new object in OpenNebula
#
# @param template [Hash] Object attributes
#
# @return [Integer] Resource ID
def create(template)
def create
# create ONE object
new_object
rc = @one.allocate(format_template(template))
rc = @one.allocate(format_template(@p_template))
Utils.exception(rc)
rc = @one.info
Utils.exception(rc)

4
src/oneprovision/lib/provision/resources/virtual/vntemplate.rb

@ -22,7 +22,9 @@ module OneProvision
class VnTemplate < VirtualResource
# Class constructor
def initialize
#
# @param p_template [Hash] Resource information in hash form
def initialize(p_template)
super
@pool = OpenNebula::VNTemplatePool.new(@client)

339
src/oneprovision/lib/provision/utils.rb

@ -15,7 +15,6 @@
#--------------------------------------------------------------------------- #
require 'yaml'
require 'erb'
require 'nokogiri'
require 'open3'
require 'tempfile'
@ -50,290 +49,6 @@ module OneProvision
ERROR_OPEN = 'ERROR MESSAGE --8<------'
ERROR_CLOSE = 'ERROR MESSAGE ------>8--'
# Validates the configuration file
#
# @param config [String] Path to the configuration file
# @param dump [Boolean] True to show the result in the console
def validate_configuration(config, dump)
config = read_config(config)
config = config.delete_if {|_k, v| v.nil? }
check_config(config)
puts config.to_yaml if dump
0
end
# Checks configuration fole
#
# @param config [Hash] Configuration content
def check_config(config)
name = config['name']
version = config['version']
if !version.nil? && version != 1
Utils.fail('There is an error in your configuration ' \
'file: Unsupported version')
end
if name.nil?
Utils.fail('There is an error in your configuration ' \
'file: no name given')
end
if !config['cluster']
Utils.fail('There is an error in your configuration ' \
'file: no cluster given')
end
if !config['cluster']['name']
Utils.fail('There is an error in your configuration ' \
'file: no cluster name given')
end
if config['hosts']
config['hosts'].each_with_index do |h, i|
im = h['im_mad']
vm = h['vm_mad']
name = h['provision']['hostname']
if im.nil?
Utils.fail('There is an error in your ' \
'configuration file: there is ' \
"no im_mad in host #{i + 1}")
end
if vm.nil?
Utils.fail('There is an error in your ' \
'configuration file: there is ' \
"no vm_mad in host #{i + 1}")
end
if name.nil?
Utils.fail('There is an error in your ' \
'configuration file: there is ' \
"no hostname in host #{i + 1}")
end
next
end
end
if config['datastores']
config['datastores'].each_with_index do |d, i|
if d['tm_mad'].nil?
Utils.fail('There is an error in your ' \
'configuration file: there is '\
"no tm_mad in datastore #{i + 1}")
end
next
end
end
return unless config['networks']
config['networks'].each_with_index do |n, i|
if n['vn_mad'].nil?
Utils.fail('There is an error in your ' \
'configuration file: there is '\
"no vn_mad in newtork #{i + 1}")
end
next
end
end
# Creates configuration
#
# @param yaml [Hash] Configuration content
#
# @return [Hash] Configuration for drivers
def create_config(yaml)
begin
check_config(yaml)
cluster = yaml['cluster']
yaml['cluster'] = { 'name' => yaml['name'] } if cluster.nil?
defaults = yaml['defaults']
# TODO: schema check
if yaml['hosts']
yaml['hosts'] = yaml['hosts'].map do |host|
sections = %w[connection provision configuration]
sections.each do |section|
data = CONFIG_DEFAULTS[section] || {}
if yaml['defaults']
defaults = yaml['defaults'][section]
end
h_sec = host[section]
# merge defaults with globals
# and device specific params
data.merge!(defaults) unless defaults.nil?
data.merge!(h_sec) unless h_sec.nil?
host[section] = data
end
host
end
end
%w[datastores networks].each do |r|
next unless yaml[r]
yaml[r] = yaml[r].map do |x|
x['provision'] ||= {}
if defaults && defaults.key?('provision')
x['provision'].merge!(defaults['provision'])
end
x
end
end
yaml['cluster']['provision'] ||= {}
if defaults && defaults.key?('provision')
yaml['cluster']['provision']
.merge!(defaults['provision'])
end
rescue StandardError => e
Utils.fail("Failed to read configuration: #{e}")
end
yaml
end
# Reads configuration content
#
# @param name [String] Path to the configuration file
#
# @return [Hash] Configuration content
def read_config(name)
begin
yaml = YAML.load_file(name)
rescue StandardError => e
Utils.fail("Failed to read template: #{e}")
end
if yaml['extends']
yaml['extends'] = [yaml['extends']].flatten
yaml['extends'].reverse.each do |f|
base = read_config(f)
yaml.delete('extends')
base['defaults'] ||= {}
yaml['defaults'] ||= {}
if base['playbook']
playbooks = []
playbooks << base['playbook']
playbooks << yaml['playbook'] if yaml['playbook']
playbooks.flatten!
yaml['playbook'] = playbooks
base.delete('playbook')
end
if yaml['playbook']
yaml['playbook'] = [yaml['playbook']]
yaml['playbook'].flatten!
end
# replace scalars or append array from child YAML
yaml.each do |key, value|
next if key == 'defaults'
if (value.is_a? Array) && (base[key].is_a? Array)
base[key].concat(value)
else
base[key] = value
end
end
# merge each defaults section separately
%w[connection provision configuration].each do |section|
base['defaults'][section] ||= {}
yaml['defaults'][section] ||= {}
defaults = yaml['defaults'][section]
base['defaults'][section].merge!(defaults)
end
yaml = base
end
end
yaml
end
# Gets the value of an ERB expression
#
# @param provision [OneProvision::Provision] Provision object
# @value [String] Value to evaluate
#
# @return [String] Evaluated value
def get_erb_value(provision, value)
unless value.match(/@./)
raise OneProvisionLoopException,
"value #{value} not allowed"
end
template = ERB.new value
begin
ret = template.result(provision._binding)
if ret.empty?
raise OneProvisionLoopException, "#{value} not found."
end
ret
rescue StandardError
raise OneProvisionLoopException, "#{value} not found."
end
end
# Evaluates ERB values
#
# @param provision [OneProvision::Provision] Provision object
# @param root [Hash] Hash with values to evaluate
#
# @return [Hash] Hash with evaluated values
def evaluate_erb(provision, root)
if root.is_a? Hash
root.each_pair do |key, value|
case value
when Array
root[key] = value.map do |x|
evaluate_erb(provision, x)
end
when Hash
root[key] = evaluate_erb(provision, value)
when String
if value =~ /<%= /
root[key] = get_erb_value(provision, value)
end
end
end
else
root = root.map {|x| evaluate_erb(provision, x) }
end
root
end
# Checks if the file can be read
#
# @param name [String] Path to file to read
@ -343,60 +58,6 @@ module OneProvision
name
end
# Creates the host deployment file
#
# @param host [Hash] Hash with host information
#
# @return [Nokogiri::XML] XML with the host information
def create_deployment_file(host)
ssh_key = try_read_file(host['connection']['public_key'])
config = Base64.strict_encode64(host['configuration'].to_yaml)
reject = %w[im_mad vm_mad provision connection configuration]
Nokogiri::XML::Builder.new do |xml|
xml.HOST do
xml.NAME "provision-#{SecureRandom.hex(24)}"
xml.TEMPLATE do
xml.IM_MAD host['im_mad']
xml.VM_MAD host['vm_mad']
xml.PROVISION do
host['provision'].each do |key, value|
if key != 'provider'
xml.send(key.upcase, value)
end
end
end
if host['configuration']
xml.PROVISION_CONFIGURATION_BASE64 config
end
if host['connection']
xml.PROVISION_CONNECTION do
host['connection'].each do |key, value|
xml.send(key.upcase, value)
end
end
end
if host['connection']
xml.CONTEXT do
if host['connection']['public_key']
xml.SSH_PUBLIC_KEY ssh_key
end
end
end
host.each do |key, value|
next if reject.include?(key)
xml.send(key.upcase, value)
end
end
end
end.doc.root
end
# Shows and error message and exit with fail code
#
# @param text [String] Error message

2
src/oneprovision/lib/terraform/providers/templates/aws/cluster.erb

@ -1,3 +1,4 @@
<% if provision['CIDR'] && provision['CIDR'] != "" %>
resource "aws_vpc" "device_<%= obj['ID'] %>" {
cidr_block = "<%= provision['CIDR'] %>"
@ -19,4 +20,5 @@ resource "aws_route" "device_<%= obj['ID'] %>" {
destination_cidr_block = "<%= provision['DEST_CIDR'] %>"
gateway_id = aws_internet_gateway.device_<%= obj['ID'] %>.id
}
<% end %>

2
src/oneprovision/lib/terraform/providers/templates/aws/network.erb

@ -1,3 +1,4 @@
<% if provision['SUB_CIDR'] && provision['SUB_CIDR'] != "" %>
resource "aws_subnet" "device_<%= obj['ID'] %>" {
vpc_id = aws_vpc.device_<%= c['ID'] %>.id
cidr_block = "<%= provision['SUB_CIDR'] %>"
@ -7,4 +8,5 @@ resource "aws_subnet" "device_<%= obj['ID'] %>" {
Name = "<%= obj['NAME'] %>_subnet"
}
}
<% end %>