1
0
mirror of https://github.com/OpenNebula/one.git synced 2025-04-01 06:50:25 +03:00

F #5155: Support more objects in provision Terraform (#398)

This commit is contained in:
Alejandro Huertas Herrero 2020-11-06 11:09:28 +01:00 committed by GitHub
parent 514ad55874
commit 415e138c8a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 374 additions and 122 deletions

View File

@ -295,6 +295,8 @@ LIB_DIRS="$LIB_LOCATION/ruby \
$LIB_LOCATION/oneprovision/lib/terraform \
$LIB_LOCATION/oneprovision/lib/terraform/providers \
$LIB_LOCATION/oneprovision/lib/terraform/providers/templates \
$LIB_LOCATION/oneprovision/lib/terraform/providers/templates/aws \
$LIB_LOCATION/oneprovision/lib/terraform/providers/templates/packet \
$LIB_LOCATION/oneprovision/lib/provision \
$LIB_LOCATION/oneprovision/lib/provision_template \
$LIB_LOCATION/oneprovision/lib/provider \
@ -726,7 +728,8 @@ INSTALL_ONEPROVISION_FILES=(
ONEPROVISION_LIB_FILES:$LIB_LOCATION/oneprovision/lib
ONEPROVISION_LIB_TF_FILES:$LIB_LOCATION/oneprovision/lib/terraform
ONEPROVISION_LIB_PROVIDERS_FILES:$LIB_LOCATION/oneprovision/lib/terraform/providers
ONEPROVISION_LIB_PROVIDERS_ERB_FILES:$LIB_LOCATION/oneprovision/lib/terraform/providers/templates
ONEPROVISION_LIB_AWS_ERB_FILES:$LIB_LOCATION/oneprovision/lib/terraform/providers/templates/aws
ONEPROVISION_LIB_PACKET_ERB_FILES:$LIB_LOCATION/oneprovision/lib/terraform/providers/templates/packet
ONEPROVISION_LIB_PROVISION_FILES:$LIB_LOCATION/oneprovision/lib/provision
ONEPROVISION_LIB_RESOURCES_FILES:$LIB_LOCATION/oneprovision/lib/provision/resources
ONEPROVISION_LIB_PHYSICAL_R_FILES:$LIB_LOCATION/oneprovision/lib/provision/resources/physical
@ -2394,10 +2397,17 @@ ONEPROVISION_LIB_PROVIDERS_FILES="src/oneprovision/lib/terraform/providers/aws.r
src/oneprovision/lib/terraform/providers/dummy.rb \
src/oneprovision/lib/terraform/providers/packet.rb"
ONEPROVISION_LIB_PROVIDERS_ERB_FILES="src/oneprovision/lib/terraform/providers/templates/aws.erb \
src/oneprovision/lib/terraform/providers/templates/aws_device.erb \
src/oneprovision/lib/terraform/providers/templates/packet.erb \
src/oneprovision/lib/terraform/providers/templates/packet_device.erb"
ONEPROVISION_LIB_AWS_ERB_FILES="src/oneprovision/lib/terraform/providers/templates/aws/cluster.erb \
src/oneprovision/lib/terraform/providers/templates/aws/datastore.erb \
src/oneprovision/lib/terraform/providers/templates/aws/host.erb \
src/oneprovision/lib/terraform/providers/templates/aws/network.erb \
src/oneprovision/lib/terraform/providers/templates/aws/provider.erb"
ONEPROVISION_LIB_PACKET_ERB_FILES="src/oneprovision/lib/terraform/providers/templates/packet/cluster.erb \
src/oneprovision/lib/terraform/providers/templates/packet/datastore.erb \
src/oneprovision/lib/terraform/providers/templates/packet/host.erb \
src/oneprovision/lib/terraform/providers/templates/packet/network.erb \
src/oneprovision/lib/terraform/providers/templates/packet/provider.erb"
#-----------------------------------------------------------------------------
# Sunstone files

View File

@ -452,7 +452,7 @@ class OneProvisionHelper < OpenNebulaHelper::OneHelper
return rc if OpenNebula.is_error?(rc)
id = host['ID']
host = OneProvision::Host.new(provision.provider)
host = OneProvision::Host.new(provision.provider['NAME'])
host.info(id)
case operation[:operation]

View File

@ -223,7 +223,6 @@ module OneProvision
rescue StandardError => e
raise e
end
when /^fatal: \[([^\]]+)\]: .* =>/i
host = Regexp.last_match(1)
end

View File

@ -185,32 +185,15 @@ module OneProvision
# Execute action with Terraform
#
# @param provider [Provider] Provider to perform actions
# @param client [Client] OpenNebula client to make calls
# @param hosts [Array] Hosts to perform action
# @param action [String] Action to perform
# @param tf [Hash] Terraform :state and :conf
def tf_action(provider, hosts, action, tf = {})
hosts = hosts.map do |h|
host = Host.new(provider)
rc = host.info(h['id'])
if OpenNebula.is_error?(rc)
raise OneProvisionLoopException, rc.message
end
host.one
# Decrypt secrets
host.one.info(true)
host.one
end
# @param provision [Provision] Provision information
# @param action [String] Action to perform
# @param tf [Hash] Terraform :state and :conf
def tf_action(provision, action, tf = {})
provider = provision.provider
terraform = Terraform.singleton(provider, tf)
terraform.conn = provider.connection
terraform.generate_deployment_file(hosts)
terraform.generate_deployment_file(provision)
terraform.send(action)
end

View File

@ -114,7 +114,7 @@ module OneProvision
# Returns provision provider
def provider
@body['provider']
Provider.by_name(@client, @body['provider'])
end
# Returns infrastructure + resource objects
@ -139,6 +139,42 @@ module OneProvision
{ :state => tf_state, :conf => tf_conf }
end
# Get OpenNebula information for specific objects
#
# @param object [String] Object to check
#
# @return [Array] OpenNebula objects
def info_objects(object)
rc = info(true)
if OpenNebula.is_error?(rc)
raise OneProvisionLoopException, rc.message
end
if FULL_CLUSTER.include?(object)
path = 'infrastructure'
else
path = 'resource'
end
return [] unless @body['provision'][path][object]
resource = Resource.object(object)
ret = []
@body['provision'][path][object].each do |o|
rc = resource.info(o['id'])
if OpenNebula.is_error?(rc)
raise OneProvisionLoopException, rc.message
end
ret << resource.one
end
ret
end
# Deploys a new provision
#
# @param config [String] Path to deployment file
@ -230,8 +266,7 @@ module OneProvision
conf = nil
Driver.retry_loop 'Failed to deploy hosts' do
ips, ids, state, conf = Driver.tf_action(provider,
hosts,
ips, ids, state, conf = Driver.tf_action(self,
'deploy')
end
@ -334,16 +369,14 @@ module OneProvision
OneProvisionLogger.info("Deleting provision #{self['ID']}")
provider = Provider.by_name(@client, provider())
if !hosts.empty? && tf_state && tf_conf
Driver.tf_action(provider, hosts, 'destroy', tf)
Driver.tf_action(self, 'destroy', tf)
end
Driver.retry_loop 'Failed to delete hosts' do
hosts.each do |host|
id = host['id']
host = Host.new(provider)
host = Host.new(provider['NAME'])
host.info(id)
host.delete
@ -388,14 +421,12 @@ module OneProvision
if operation == :append
@body['provision'][path][object] << { :id => id, :name => name }
else
provider = Provider.by_name(@client, provider())
o = Resource.object(object, provider)
rc = o.info(id)
return rc if OpenNebula.is_error?(rc)
rc = o.delete(object == 'hosts' ? tf : nil)
rc = o.delete(FULL_CLUSTER.include?(object) ? tf : nil)
return rc if OpenNebula.is_error?(rc)

View File

@ -22,11 +22,14 @@ module OneProvision
class Cluster < Resource
# Class constructor
def initialize
super
#
# @param provider [provider] Cluster provider
def initialize(provider = nil)
super()
@pool = OpenNebula::ClusterPool.new(@client)
@type = 'cluster'
@pool = OpenNebula::ClusterPool.new(@client)
@type = 'cluster'
@provider = provider
end
# Creates a new cluster in OpenNebula
@ -52,7 +55,29 @@ module OneProvision
# @param id [Integer] Object ID
def info(id)
@one = OpenNebula::Cluster.new_with_id(id, @client)
@one.info
@one.info(true)
end
# Deletes the cluster
#
# @param tf [Hash] Terraform :conf and :state
#
# @return [Array]
# - Terraform state in base64
# - Terraform config in base64
def delete(tf = nil)
if tf && !tf.empty?
terraform = Terraform.singleton(@provider, tf)
state, conf = terraform.destroy_cluster(@one.id)
end
Utils.exception(@one.delete)
if state && conf
[state, conf]
else
0
end
end
private

View File

@ -22,7 +22,9 @@ module OneProvision
class Datastore < PhysicalResource
# Class constructor
def initialize
#
# @param provider [Provider] Datastore provider
def initialize(provider = nil)
super
@pool = OpenNebula::DatastorePool.new(@client)
@ -34,7 +36,15 @@ module OneProvision
# @param id [String] Object ID
def info(id)
@one = OpenNebula::Datastore.new_with_id(id, @client)
@one.info
@one.info(true)
end
# Destroy datastore in provider
#
# @param tf [Hash] Terraform configuration
def destroy(tf)
terraform = Terraform.singleton(@provider, tf)
terraform.destroy_datastore(@one.id)
end
private

View File

@ -163,7 +163,7 @@ module OneProvision
# @param id [String] Object ID
def info(id)
@one = OpenNebula::Host.new_with_id(id, @client)
@one.info
@one.info(true)
end
private

View File

@ -22,7 +22,9 @@ module OneProvision
class Network < PhysicalResource
# Class constructor
def initialize
#
# @param provider [provider] Network provider
def initialize(provider = nil)
super
@pool = OpenNebula::VirtualNetworkPool.new(@client)
@ -34,7 +36,15 @@ module OneProvision
# @param id [String] Object ID
def info(id)
@one = OpenNebula::VirtualNetwork.new_with_id(id, @client)
@one.info
@one.info(true)
end
# Destroy network in provider
#
# @param tf [Hash] Terraform configuration
def destroy(tf)
terraform = Terraform.singleton(@provider, tf)
terraform.destroy_network(@one.id)
end
private

View File

@ -21,6 +21,15 @@ module OneProvision
# Represents a physical resource for the provision
class PhysicalResource < Resource
# Class constructor
#
# @param provider [Provider] Resource provider
def initialize(provider)
super()
@provider = provider
end
# Creates the object in OpenNebula
#
# @param template [Hash] Object attributes
@ -43,6 +52,25 @@ module OneProvision
Integer(@one.id)
end
# Deletes the resource
#
# @param tf [Hash] Terraform :conf and :state
#
# @return [Array]
# - Terraform state in base64
# - Terraform config in base64
def delete(tf = nil)
state, conf = destroy(tf) if tf && !tf.empty?
Utils.exception(@one.delete)
if state && conf
[state, conf]
else
0
end
end
end
end

View File

@ -45,15 +45,15 @@ module OneProvision
case type
when :cluster
Cluster.new
Cluster.new(provider)
when :datastore
Datastore.new
when :host
Host.new(provider)
when :image
Image.new
Image.new(provider)
when :network
Network.new
Network.new(provider)
when :template
Template.new
when :vntemplate

View File

@ -37,7 +37,7 @@ module OneProvision
# @param id [String] Object ID
def info(id)
@one = OpenNebula::ServiceTemplate.new_with_id(id, @client)
@one.info
@one.info(true)
end
# Get template in json format

View File

@ -68,7 +68,7 @@ module OneProvision
# @param id [String] Object ID
def info(id)
@one = OpenNebula::Image.new_with_id(id, @client)
@one.info
@one.info(true)
end
private

View File

@ -34,7 +34,7 @@ module OneProvision
# @param id [String] Object ID
def info(id)
@one = OpenNebula::Template.new_with_id(id, @client)
@one.info
@one.info(true)
end
private

View File

@ -34,7 +34,7 @@ module OneProvision
# @param id [String] Object ID
def info(id)
@one = OpenNebula::VNTemplate.new_with_id(id, @client)
@one.info
@one.info(true)
end
private

View File

@ -22,14 +22,20 @@ module OneProvision
# AWS Terraform Provider
class AWS < Terraform
# OpenNebula - Terraform equivalence
TYPES = {
:cluster => 'aws_vpc',
:datastore => 'aws_ebs_volume',
:host => 'aws_instance',
:network => 'aws_subnet'
}
# Class constructor
#
# @param state [String] Terraform state in base64
# @param conf [String] Terraform config state in base64
def initialize(state, conf)
@device_erb = "#{PROVIDERS_LOCATION}/templates/aws_device.erb"
@erb = "#{PROVIDERS_LOCATION}/templates/aws.erb"
@host_type = 'aws_instance'
@dir = "#{PROVIDERS_LOCATION}/templates/aws"
# User data should be encoded in base64
@base64 = true

View File

@ -22,14 +22,19 @@ module OneProvision
# Packet Terraform Provider
class Packet < Terraform
# OpenNebula - Terraform equivalence
TYPES = {
:datastore => 'packet_volume',
:host => 'packet_device',
:network => 'packet_reserved_ip_block'
}
# Class constructor
#
# @param state [String] Terraform state in base64
# @param conf [String] Terraform config state in base64
def initialize(state, conf)
@device_erb = "#{PROVIDERS_LOCATION}/templates/packet_device.erb"
@erb = "#{PROVIDERS_LOCATION}/templates/packet.erb"
@host_type = 'packet_device'
@dir = "#{PROVIDERS_LOCATION}/templates/packet"
# User data is in plain text
@base64 = false

View File

@ -0,0 +1,22 @@
resource "aws_vpc" "device_<%= obj['ID'] %>" {
cidr_block = "<%= provision['CIDR'] %>"
tags = {
Name = "<%= obj['NAME'] %>_vpc"
}
}
resource "aws_internet_gateway" "device_<%= obj['ID'] %>" {
vpc_id = aws_vpc.device_<%= obj['ID'] %>.id
tags = {
Name = "<%= obj['NAME'] %>_gateway"
}
}
resource "aws_route" "device_<%= obj['ID'] %>" {
route_table_id = aws_vpc.device_<%= obj['ID'] %>.main_route_table_id
destination_cidr_block = "<%= provision['DEST_CIDR'] %>"
gateway_id = aws_internet_gateway.device_<%= obj['ID'] %>.id
}

View File

@ -0,0 +1,9 @@
resource "aws_ebs_volume" "device_<%= obj['ID'] %>" {
availability_zone = "<%= provision['AWS_REGION'] %>"
size = "<%= obj['TOTAL_MB'] %>"
tags = {
Name = "<%= obj['NAME'] %>"
}
}

View File

@ -1,4 +1,4 @@
resource "aws_instance" "device_<%= host['ID'] %>" {
resource "aws_instance" "device_<%= obj['ID'] %>" {
ami = "<%= provision['AMI'] %>"
instance_type = "<%= provision['INSTANCETYPE'] %>"
@ -10,17 +10,17 @@ resource "aws_instance" "device_<%= host['ID'] %>" {
vpc_security_group_ids = ["<%= provision['SECURITYGROUPSIDS'] %>"]
<% end %>
user_data = "<%= user_data %>"
user_data = "<%= obj['user_data'] %>"
tags = {
Name = "<%= provision['HOSTNAME'] %>"
}
}
output "ip_<%= host['ID'] %>" {
value = aws_instance.device_<%= host['ID'] %>.public_ip
output "ip_<%= obj['ID'] %>" {
value = aws_instance.device_<%= obj['ID'] %>.public_ip
}
output "device_id_<%= host['ID'] %>" {
value = aws_instance.device_<%= host['ID'] %>.id
output "device_id_<%= obj['ID'] %>" {
value = aws_instance.device_<%= obj['ID'] %>.id
}

View File

@ -0,0 +1,10 @@
resource "aws_subnet" "device_<%= obj['ID'] %>" {
vpc_id = aws_vpc.device_<%= c['ID'] %>.id
cidr_block = "<%= provision['SUB_CIDR'] %>"
map_public_ip_on_launch = true
tags = {
Name = "<%= obj['NAME'] %>_subnet"
}
}

View File

@ -0,0 +1,9 @@
resource "packet_volume" "device_<%= obj['ID'] %>" {
description = "<%= obj['ID'] %>_volume"
facility = "<%= provision['FACILITY'] %>"
project_id = "<%= provision['PACKET_PROJECT'] %>"
plan = "<%= provision['PLAN'] %>"
size = "<%= obj['TOTAL_MB'] %>"
billing_cycle = "hourly"
}

View File

@ -0,0 +1,19 @@
resource "packet_device" "device_<%= obj['ID'] %>" {
hostname = "<%= provision['HOSTNAME'] %>"
plan = "<%= provision['PLAN'] %>"
facilities = ["<%= provision['FACILITY'] %>"]
operating_system = "<%= provision['OS'] %>"
project_id = "<%= provision['PACKET_PROJECT']%>"
billing_cycle = "hourly"
user_data = "<%= obj['user_data'] %>"
tags = ["OpenNebula", "ONE_ID=<%= obj['ID'] %>"]
}
output "ip_<%= obj['ID'] %>" {
value = packet_device.device_<%= obj['ID'] %>.network[0].address
}
output "device_id_<%= obj['ID'] %>" {
value = packet_device.device_<%= obj['ID'] %>.id
}

View File

@ -0,0 +1,6 @@
resource "packet_reserved_ip_block" "device_<%= obj['ID'] %>" {
project_id = "<%= provision['PACKET_PROJECT'] %>"
facility = "<%= provision['FACILITY'] %>"
quantity = "<%= provision['SIZE'] %>"
}

View File

@ -1,19 +0,0 @@
resource "packet_device" "device_<%= host['ID'] %>" {
hostname = "<%= provision['HOSTNAME'] %>"
plan = "<%= provision['PLAN'] %>"
facilities = ["<%= provision['FACILITY'] %>"]
operating_system = "<%= provision['OS'] %>"
project_id = "<%= provision['PACKET_PROJECT']%>"
billing_cycle = "hourly"
user_data = "<%= user_data %>"
tags = ["OpenNebula", "ONE_ID=<%= host['ID'] %>"]
}
output "ip_<%= host['ID'] %>" {
value = packet_device.device_<%= host['ID'] %>.network[0].address
}
output "device_id_<%= host['ID'] %>" {
value = packet_device.device_<%= host['ID'] %>.id
}

View File

@ -73,48 +73,28 @@ module OneProvision
# Generate Terraform deployment file
#
# @param hosts [Array] Hosts to deploy in Packet
def generate_deployment_file(hosts)
# @param provision [Provision] Provision information
def generate_deployment_file(provision)
return if @conf
@conf = ''
c = File.read(@erb)
c = File.read("#{@dir}/provider.erb")
c = ERBVal.render_from_hash(c, :conn => @conn)
@conf << c
hosts.each do |host|
host = host.to_hash['HOST']
provision = host['TEMPLATE']['PROVISION']
ssh_key = host['TEMPLATE']['CONTEXT']['SSH_PUBLIC_KEY']
# Generate clusters Terraform configuration
cluster_info(provision)
# Add clod unit information into user_data
# This only applies for a set of spported providers
user_data = "#cloud-config\n"
# Generate hosts Terraform configuration
host_info(provision)
if ssh_key
user_data << "ssh_authorized_keys:\n"
# Generate datastores Terraform configuration
ds_info(provision)
ssh_key.split("\n").each do |key|
user_data << "- #{key}\n"
end
end
if @base64
user_data = Base64.strict_encode64(user_data)
else
# Escape \n to avoid multilines in Terraform deploy file
user_data = user_data.gsub("\n", '\\n')
end
c = File.read(@device_erb)
c = ERBVal.render_from_hash(c,
:host => host,
:provision => provision,
:user_data => user_data)
@conf << c
end
# Generate networks Terraform configuration
network_info(provision)
end
# Deploy infra via Terraform
@ -134,7 +114,7 @@ module OneProvision
)
unless s && s.success?
STDERR.puts '[ERROR] Hosts provision failed!!!' \
STDERR.puts '[ERROR] Hosts provision failed!!! ' \
'Please log in to your console to delete ' \
'left resources'
@ -209,15 +189,124 @@ module OneProvision
FileUtils.rm_r(tempdir) if File.exist?(tempdir)
end
# Destroys a cluster
#
# @param id [String] Host ID
def destroy_cluster(id)
destroy_resource(self.class::TYPES[:cluster], id)
end
# Destroys a host
#
# @param host [String] Host ID
# @param id [String] Host ID
def destroy_host(id)
destroy_resource(@host_type, id)
destroy_resource(self.class::TYPES[:host], id)
end
# Destroys a datastore
#
# @param id [String] Datastore ID
def destroy_datastore(id)
destroy_resource(self.class::TYPES[:datastore], id)
end
# Destriys a network
#
# @param id [String] Network ID
def destroy_network(id)
destroy_resource(self.class::TYPES[:network], id)
end
private
########################################################################
# Configuration file generation
########################################################################
# Add clusters information to configuration
#
# @param provision [Provision] Provision information
def cluster_info(provision)
object_info(provision, 'clusters', 'CLUSTER', 'cluster.erb')
end
# Add hosts information to configuration
#
# @param provision [Provision] Provision information
def host_info(provision)
object_info(provision, 'hosts', 'HOST', 'host.erb') do |obj|
ssh_key = obj['TEMPLATE']['CONTEXT']['SSH_PUBLIC_KEY']
return if !ssh_key || ssh_key.empty?
# Add clod unit information into user_data
# This only applies for a set of spported providers
user_data = "#cloud-config\n"
user_data << "ssh_authorized_keys:\n"
ssh_key.split("\n").each {|key| user_data << "- #{key}\n" }
if @base64
user_data = Base64.strict_encode64(user_data)
else
# Escape \n to avoid multilines in Terraform deploy file
user_data = user_data.gsub("\n", '\\n')
end
obj['user_data'] = user_data
end
end
# Add datastores information to configuration
#
# @param provision [Provision] Provision information
def ds_info(provision)
object_info(provision, 'datastores', 'DATASTORE', 'datastore.erb')
end
# Add networks information to configuration
#
# @param provision [Provision] Provision information
def network_info(provision)
object_info(provision, 'networks', 'VNET', 'network.erb')
end
# Generate object Terraform configuration
#
# @param provision [Provision] Provision information
# @param objects [String] Objects to get
# @param object [String] Object name
# @param erb [String] ERB file
def object_info(provision, objects, object, erb)
cluster = provision.info_objects('clusters')[0]
cluster = cluster.to_hash['CLUSTER']
provision.info_objects(objects).each do |obj|
obj = obj.to_hash[object]
p = obj['TEMPLATE']['PROVISION']
next if !p || p.empty?
yield(obj) if block_given?
c = File.read("#{@dir}/#{erb}")
next if c.empty?
c = ERBVal.render_from_hash(c,
:c => cluster,
:obj => obj,
:provision => p)
@conf << c
end
end
########################################################################
# Helper functions
########################################################################
# Initialize Terraform directory content
#
# @param state [Boolean] True to copy state, false otherwise