shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-01-03 01:17:41 +03:00
Code Releases Wiki Activity

F #6641: Implement OneGateProxy in VN drivers (#3142)

Browse Source 
This commit implements a transparent proxy for OneGate service (as well as any other  TCP service)
* #6281: Disable legacy OneGateProxy
* Implement OneGateProxy in VN drivers
This commit is contained in:
Michal Opala 2024-07-19 11:55:49 +02:00 committed by GitHub
parent af20fdda65
commit 360605628c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 788 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
install.sh
View File

@ -46,7 +46,6 @@ usage() {
echo "-F: install OpenNebula FireEdge"
echo "-P: do not install OpenNebula FireEdge non-minified files"
echo "-G: install only OpenNebula Gate"
echo "-6: install only OpenNebula Gate Proxy"
echo "-f: install only OpenNebula Flow"
echo "-r: remove Opennebula, only useful if -d was not specified, otherwise"
echo " rm -rf \$ONE_LOCATION would do the job"
@ -62,7 +61,6 @@ UNINSTALL="no"
LINK="no"
CLIENT="no"
ONEGATE="no"
ONEGATE_PROXY="no"
SUNSTONE="no"
SUNSTONE_DEV="yes"
FIREEDGE="no"
@ -80,7 +78,6 @@ while getopts $PARAMETERS opt; do
l) LINK="yes" ;;
c) CLIENT="yes"; INSTALL_ETC="no" ;;
G) ONEGATE="yes" ;;
6) ONEGATE_PROXY="yes" ;;
s) SUNSTONE="yes" ;;
p) SUNSTONE_DEV="no" ;;
F) FIREEDGE="yes" ;;
@ -109,7 +106,6 @@ if [ -z "$ROOT" ] ; then
LOG_LOCATION="/var/log/one"
VAR_LOCATION="/var/lib/one"
ONEGATE_LOCATION="$LIB_LOCATION/onegate"
ONEGATE_PROXY_LOCATION="$LIB_LOCATION/onegate-proxy"
SUNSTONE_LOCATION="$LIB_LOCATION/sunstone"
FIREEDGE_LOCATION="$LIB_LOCATION/fireedge"
ONEFLOW_LOCATION="$LIB_LOCATION/oneflow"
@ -151,13 +147,6 @@ if [ -z "$ROOT" ] ; then
DELETE_DIRS="$MAKE_DIRS"
CHOWN_DIRS=""
elif [ "$ONEGATE_PROXY" = "yes" ]; then
MAKE_DIRS="$BIN_LOCATION $LIB_LOCATION $VAR_LOCATION \
$ONEGATE_PROXY_LOCATION"
DELETE_DIRS="$MAKE_DIRS"
CHOWN_DIRS=""
elif [ "$ONEFLOW" = "yes" ]; then
MAKE_DIRS="$BIN_LOCATION $LIB_LOCATION $VAR_LOCATION $ONEFLOW_LOCATION \
@ -171,7 +160,7 @@ if [ -z "$ROOT" ] ; then
$INCLUDE_LOCATION $SHARE_LOCATION $DOCS_LOCATION \
$LOG_LOCATION $RUN_LOCATION $LOCK_LOCATION \
$SYSTEM_DS_LOCATION $DEFAULT_DS_LOCATION $MAN_LOCATION \
$VM_LOCATION $ONEGATE_LOCATION $ONEGATE_PROXY_LOCATION $ONEFLOW_LOCATION \
$VM_LOCATION $ONEGATE_LOCATION $ONEFLOW_LOCATION \
$SUNSTONE_MAIN_JS_LOCATION $ONEHEM_LOCATION"
DELETE_DIRS="$LIB_LOCATION $ETC_LOCATION $LOG_LOCATION $VAR_LOCATION \
@ -189,7 +178,6 @@ else
RUN_LOCATION="$VAR_LOCATION/run"
LOCK_LOCATION="$VAR_LOCATION/lock"
ONEGATE_LOCATION="$LIB_LOCATION/onegate"
ONEGATE_PROXY_LOCATION="$LIB_LOCATION/onegate-proxy"
SUNSTONE_LOCATION="$LIB_LOCATION/sunstone"
FIREEDGE_LOCATION="$LIB_LOCATION/fireedge"
ONEFLOW_LOCATION="$LIB_LOCATION/oneflow"
@ -211,11 +199,6 @@ else
MAKE_DIRS="$BIN_LOCATION $LIB_LOCATION $VAR_LOCATION \
$ONEGATE_LOCATION $ETC_LOCATION"
DELETE_DIRS="$MAKE_DIRS"
elif [ "$ONEGATE_PROXY" = "yes" ]; then
MAKE_DIRS="$BIN_LOCATION $LIB_LOCATION $VAR_LOCATION \
$ONEGATE_PROXY_LOCATION"
DELETE_DIRS="$MAKE_DIRS"
elif [ "$SUNSTONE" = "yes" ]; then
MAKE_DIRS="$BIN_LOCATION $LIB_LOCATION $VAR_LOCATION \
@ -236,7 +219,7 @@ else
MAKE_DIRS="$BIN_LOCATION $SBIN_LOCATION $LIB_LOCATION $ETC_LOCATION $VAR_LOCATION \
$INCLUDE_LOCATION $SHARE_LOCATION $SYSTEM_DS_LOCATION \
$DEFAULT_DS_LOCATION $MAN_LOCATION $DOCS_LOCATION \
$VM_LOCATION $ONEGATE_LOCATION $ONEGATE_PROXY_LOCATION $ONEFLOW_LOCATION \
$VM_LOCATION $ONEGATE_LOCATION $ONEFLOW_LOCATION \
$SUNSTONE_MAIN_JS_LOCATION $ONEHEM_LOCATION $LOCK_LOCATION $RUN_LOCATION"
DELETE_DIRS="$MAKE_DIRS"
@ -535,8 +518,6 @@ if [ "$CLIENT" = "yes" ]; then
$ETC_LOCATION"
elif [ "$ONEGATE" = "yes" ]; then
MAKE_DIRS="$MAKE_DIRS $LIB_OCA_CLIENT_DIRS"
elif [ "$ONEGATE_PROXY" = "yes" ]; then
MAKE_DIRS="$MAKE_DIRS $LIB_OCA_CLIENT_DIRS"
elif [ "$SUNSTONE" = "yes" ]; then
if [ "$SUNSTONE_DEV" = "no" ]; then
MAKE_DIRS="$MAKE_DIRS $SUNSTONE_DIRS $SUNSTONE_MINIFIED_DIRS $LIB_OCA_CLIENT_DIRS"
@ -875,15 +856,6 @@ INSTALL_ONEGATE_ETC_FILES=(
ONEGATE_ETC_FILES:$ETC_LOCATION
)
INSTALL_ONEGATE_PROXY_FILES=(
ONEGATE_PROXY_FILES:$ONEGATE_PROXY_LOCATION
ONEGATE_PROXY_BIN_FILES:$BIN_LOCATION
)
INSTALL_ONEGATE_PROXY_ETC_FILES=(
ONEGATE_PROXY_REMOTES_ETC_FILES:$VAR_LOCATION/remotes/etc
)
INSTALL_ONEFLOW_FILES=(
ONEFLOW_FILES:$ONEFLOW_LOCATION
ONEFLOW_BIN_FILES:$BIN_LOCATION
@ -1566,7 +1538,9 @@ NETWORK_FILES="src/vnm_mad/remotes/lib/vnm_driver.rb \
src/vnm_mad/remotes/lib/no_vlan.rb \
src/vnm_mad/remotes/lib/security_groups.rb \
src/vnm_mad/remotes/lib/security_groups_iptables.rb \
src/vnm_mad/remotes/lib/nic.rb"
src/vnm_mad/remotes/lib/nic.rb \
src/vnm_mad/remotes/lib/tproxy \
src/vnm_mad/remotes/lib/tproxy.rb"
NETWORK_8021Q_FILES="src/vnm_mad/remotes/802.1Q/clean \
src/vnm_mad/remotes/802.1Q/post \
@ -2802,16 +2776,6 @@ ONEGATE_BIN_FILES="src/onegate/bin/onegate-server"
ONEGATE_ETC_FILES="src/onegate/etc/onegate-server.conf"
#-----------------------------------------------------------------------------
# OneGateProxy files
#-----------------------------------------------------------------------------
ONEGATE_PROXY_FILES="src/onegate-proxy/onegate-proxy.rb"
ONEGATE_PROXY_BIN_FILES="src/onegate-proxy/bin/onegate-proxy"
ONEGATE_PROXY_REMOTES_ETC_FILES="src/onegate-proxy/etc/onegate-proxy.conf"
#-----------------------------------------------------------------------------
# OneFlow files
#-----------------------------------------------------------------------------
@ -3046,8 +3010,6 @@ if [ "$CLIENT" = "yes" ]; then
INSTALL_SET=${INSTALL_CLIENT_FILES[@]}
elif [ "$ONEGATE" = "yes" ]; then
INSTALL_SET="${INSTALL_ONEGATE_FILES[@]}"
elif [ "$ONEGATE_PROXY" = "yes" ]; then
INSTALL_SET="${INSTALL_ONEGATE_PROXY_FILES[@]}"
elif [ "$SUNSTONE" = "yes" ]; then
if [ "$SUNSTONE_DEV" = "no" ]; then
INSTALL_SET="${INSTALL_SUNSTONE_RUBY_FILES[@]} \
@ -3088,7 +3050,6 @@ else
${INSTALL_SUNSTONE_FILES[@]} ${INSTALL_SUNSTONE_PUBLIC_DEV_DIR[@]}\
${INSTALL_FIREEDGE_FILES[@]} ${INSTALL_FIREEDGE_DEV_DIRS[@]}\
${INSTALL_ONEGATE_FILES[@]} \
${INSTALL_ONEGATE_PROXY_FILES[@]} \
${INSTALL_ONEFLOW_FILES[@]} \
${INSTALL_ONEHEM_FILES[@]} \
${INSTALL_ONEPROVISION_FILES[@]} \
@ -3113,8 +3074,6 @@ if [ "$INSTALL_ETC" = "yes" ] ; then
INSTALL_ETC_SET="${INSTALL_FIREEDGE_ETC_FILES[@]}"
elif [ "$ONEGATE" = "yes" ]; then
INSTALL_ETC_SET="${INSTALL_ONEGATE_ETC_FILES[@]}"
elif [ "$ONEGATE_PROXY" = "yes" ]; then
INSTALL_ETC_SET="${INSTALL_ONEGATE_PROXY_ETC_FILES[@]}"
elif [ "$ONEFLOW" = "yes" ]; then
INSTALL_ETC_SET="${INSTALL_ONEFLOW_ETC_FILES[@]}"
else
@ -3122,7 +3081,6 @@ if [ "$INSTALL_ETC" = "yes" ] ; then
${INSTALL_SUNSTONE_ETC_FILES[@]} \
${INSTALL_FIREEDGE_ETC_FILES[@]} \
${INSTALL_ONEGATE_ETC_FILES[@]} \
${INSTALL_ONEGATE_PROXY_ETC_FILES[@]} \
${INSTALL_ONEHEM_ETC_FILES[@]} \
${INSTALL_ONEFLOW_ETC_FILES[@]}"
fi

11
share/pkgs/logrotate/opennebula-gate-proxy
View File

@ -1,11 +0,0 @@
/var/log/one/onegate-proxy.log {
delaycompress
dateext
dateformat -%Y%m%d-%s
compress
weekly
rotate 52
missingok
notifempty
copytruncate
}

15
share/pkgs/services/systemd/opennebula-gate-proxy.service
View File

@ -1,15 +0,0 @@
[Unit]
Description=OpenNebula Gate Proxy Service
After=syslog.target network.target
[Service]
Type=simple
Group=root
User=root
ExecStart=/usr/bin/ruby /usr/lib/one/onegate-proxy/onegate-proxy.rb
Restart=always
RestartSec=5
SyslogIdentifier=opennebula-gate-proxy
[Install]
WantedBy=multi-user.target

2
share/pkgs/sudoers/centos/opennebula
View File

@ -6,7 +6,7 @@ Cmnd_Alias ONE_HA = /usr/bin/systemctl start opennebula-flow, /usr/bin/systemctl
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip tuntap *, /usr/sbin/ip route *, /usr/sbin/ip neighbour *
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *, /usr/sbin/nft, /usr/sbin/sysctl net.ipv4.conf.*, /var/tmp/one/vnm/tproxy
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu

2
share/pkgs/sudoers/debian/opennebula
View File

@ -6,7 +6,7 @@ Cmnd_Alias ONE_HA = /usr/bin/systemctl start opennebula-flow, /usr/bin/systemctl
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip tuntap *, /usr/sbin/ip route *, /usr/sbin/ip neighbour *
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *, /usr/sbin/nft, /usr/sbin/sysctl net.ipv4.conf.*, /var/tmp/one/vnm/tproxy
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu

8
share/sudoers/sudoers.rb
View File

@ -30,9 +30,13 @@ class Sudoers
'ip6tables',
'ipset',
'ip link *',
'ip tuntap *',
'ip neighbour *',
'ip route *',
'ip neighbour *'
'ip rule *',
'ip tuntap *',
'nft',
'sysctl net.ipv4.conf.*',
'/var/tmp/one/vnm/tproxy'
],
:LVM => [
'lvcreate', 'lvremove', 'lvs', 'vgdisplay', 'lvchange', 'lvscan', 'lvextend'

120
src/onegate-proxy/bin/onegate-proxy
View File

@ -1,120 +0,0 @@
#!/bin/bash
# -------------------------------------------------------------------------- #
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
if [ -z "$ONE_LOCATION" ]; then
ONEGATE_PROXY_PID=/var/run/one/onegate-proxy.pid
ONEGATE_PROXY_SERVER=/usr/lib/one/onegate-proxy/onegate-proxy.rb
ONEGATE_PROXY_LOCK_FILE=/var/lock/one/.onegate-proxy.lock
ONEGATE_PROXY_LOG=/var/log/one/onegate-proxy.log
ONEGATE_PROXY_LOG_ERROR=/var/log/one/onegate-proxy.error
else
ONEGATE_PROXY_PID=$ONE_LOCATION/var/onegate-proxy.pid
ONEGATE_PROXY_SERVER=$ONE_LOCATION/lib/onegate-proxy/onegate-proxy.rb
ONEGATE_PROXY_LOCK_FILE=$ONE_LOCATION/var/.onegate-proxy.lock
ONEGATE_PROXY_LOG=$ONE_LOCATION/var/onegate-proxy.log
ONEGATE_PROXY_LOG_ERROR=$ONE_LOCATION/var/onegate-proxy.error
fi
setup()
{
if [ -f $ONEGATE_PROXY_LOCK_FILE ]; then
if [ -f $ONEGATE_PROXY_PID ]; then
ONEGATEPID=`cat $ONEGATE_PROXY_PID`
ps $ONEGATEPID &> /dev/null
if [ $? -eq 0 ]; then
echo -n "OneGate Server is still running (PID:$ONEGATEPID). Please "
echo "try 'onegate-proxy stop' first."
exit 1
fi
fi
echo "Stale .lock detected. Erasing it."
rm $ONEGATE_PROXY_LOCK_FILE
fi
}
start()
{
if [ ! -f "$ONEGATE_PROXY_SERVER" ]; then
echo "Cannot find $ONEGATE_PROXY_SERVER."
exit 1
fi
# Start the onegate daemon
touch $ONEGATE_PROXY_LOCK_FILE
ruby $ONEGATE_PROXY_SERVER > $ONEGATE_PROXY_LOG 2>$ONEGATE_PROXY_LOG_ERROR &
LASTPID=$!
if [ $? -ne 0 ]; then
echo "Error executing onegate-proxy."
echo "Check $ONEGATE_PROXY_LOG_ERROR and $ONEGATE_PROXY_LOG for more information"
exit 1
else
echo $LASTPID > $ONEGATE_PROXY_PID
fi
sleep 1
ps $LASTPID &> /dev/null
if [ $? -ne 0 ]; then
echo "Error executing onegate-proxy."
echo "Check $ONEGATE_PROXY_LOG_ERROR and $ONEGATE_PROXY_LOG for more information"
exit 1
fi
echo "onegate-proxy started"
}
#
# Function that stops the daemon/service
#
stop()
{
if [ ! -f $ONEGATE_PROXY_PID ]; then
echo "Couldn't find onegate-proxy process pid."
exit 1
fi
# Kill the onegate daemon
kill -INT `cat $ONEGATE_PROXY_PID` &> /dev/null
# Remove pid files
rm -f $ONEGATE_PROXY_LOCK_FILE &> /dev/null
rm -f $ONEGATE_PROXY_PID &> /dev/null
echo "onegate-proxy stopped"
}
case "$1" in
start)
setup
start
;;
stop)
stop
;;
restart)
stop
setup
start
;;
*)
echo "Usage: onegate-proxy {start|stop|restart}" >&2
exit 3
;;
esac

11
src/onegate-proxy/etc/onegate-proxy.conf
View File

@ -1,11 +0,0 @@
# Log debug level: 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
#:debug_level: 2
#:process_owner: 'oneadmin'
# The address and port are supposed to be provided during onehost sync
# operation inside the '/var/tmp/one/etc/onegate-proxy.conf' file.
#:onegate_addr: 'localhost'
#:onegate_port: '5030'
#:service_addr: '169.254.16.9'

309
src/onegate-proxy/onegate-proxy.rb
View File

@ -1,309 +0,0 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
# frozen_string_literal: true
# rubocop:disable Lint/MissingCopEnableDirective
# rubocop:disable Lint/RedundantRequireStatement
# rubocop:disable Layout/FirstHashElementIndentation
# rubocop:disable Layout/HashAlignment
# rubocop:disable Layout/HeredocIndentation
# rubocop:disable Layout/IndentationWidth
# rubocop:disable Style/HashSyntax
# rubocop:disable Style/ParallelAssignment
ONE_LOCATION = ENV['ONE_LOCATION']
if !ONE_LOCATION
RUBY_LIB_LOCATION = '/usr/lib/one/ruby'
GEMS_LOCATION = '/usr/share/one/gems'
ETC_LOCATION = '/etc/one'
REMOTES_LOCATION = '/var/tmp/one'
else
RUBY_LIB_LOCATION = ONE_LOCATION + '/lib/ruby'
GEMS_LOCATION = ONE_LOCATION + '/share/gems'
ETC_LOCATION = ONE_LOCATION + '/etc'
REMOTES_LOCATION = ONE_LOCATION + '/var/remotes'
end
CONFIGURATION_FILE = REMOTES_LOCATION + '/etc/onegate-proxy.conf'
# %%RUBYGEMS_SETUP_BEGIN%%
if File.directory?(GEMS_LOCATION)
real_gems_path = File.realpath(GEMS_LOCATION)
if !defined?(Gem) || Gem.path != [real_gems_path]
$LOAD_PATH.reject! {|p| p =~ /vendor_ruby/ }
# Suppress warnings from Rubygems
# https://github.com/OpenNebula/one/issues/5379
begin
verb = $VERBOSE
$VERBOSE = nil
require 'rubygems'
Gem.use_paths(real_gems_path)
ensure
$VERBOSE = verb
end
end
end
# %%RUBYGEMS_SETUP_END%%
$LOAD_PATH << RUBY_LIB_LOCATION
require 'async/io'
require 'async/io/stream'
require 'async/io/trap'
require 'etc'
require 'pp'
require 'rb-inotify'
require 'socket'
require 'yaml'
$stdout.sync = true
$stderr.sync = true
DEFAULT_OPTIONS = {
:debug_level => 2, # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
:process_owner => 'oneadmin',
:onegate_addr => '127.0.0.1',
:onegate_port => '5030',
:service_addr => '169.254.16.9'
}.freeze
# Proxy-class for converting log levels between OpenNebula and
# the socketry/console library. It also splits specific log levels
# into separate stdout and stderr loggers.
class Logger
LOG_LEVEL_MAP = {
0 => '3', # ERROR
1 => '2', # WARN
2 => '1', # INFO
3 => '0' # DEBUG
}.freeze
def initialize(log_level = 2)
@out = Console::Logger.default_logger $stdout, {
'CONSOLE_LEVEL' => LOG_LEVEL_MAP[log_level]
}
@err = Console::Logger.default_logger $stderr, {
'CONSOLE_LEVEL' => LOG_LEVEL_MAP[log_level]
}
end
# rubocop:disable Style/ArgumentsForwarding
def error(*args, &block)
@err.error(*args, &block)
end
def warn(*args, &block)
@err.warn(*args, &block)
end
def info(*args, &block)
@out.info(*args, &block)
end
def debug(*args, &block)
@err.debug(*args, &block)
end
# rubocop:enable Style/ArgumentsForwarding
end
# Class that implements a classic two-way TCP socket proxy (async).
class OneGateProxy
def initialize(options = {})
@options = DEFAULT_OPTIONS.dup.merge! options
@options.each {|k, v| instance_variable_set("@#{k}", v) }
@logger = Logger.new options[:debug_level]
@sigint = Async::IO::Trap.new :INT
@sigint.install!
@inotify = setup_inotify
@inotify_io = Async::IO::Generic.new @inotify.to_io
@proxy_ep = Async::IO::Endpoint.socket setup_socket
end
def run
# NOTE: At this point all config should be set in stone,
# we can drop root privileges..
drop_privileges
Async do |task|
# Make CTRL-C work..
task.async do
@sigint.wait { exit 0 }
end
# Handle filesystem notifications..
task.async do
@inotify.process while @inotify_io.wait_readable
end
glue_peers task
end
end
private
def drop_privileges
new_gid, new_uid = Etc.getpwnam(@process_owner).gid,
Etc.getpwnam(@process_owner).uid
@logger.info(self) do
"Drop root privileges -> #{@process_owner}"
end
Process::Sys.setgid new_gid
Process::Sys.setuid new_uid
end
def setup_inotify
inotify = INotify::Notifier.new
inotify.watch(CONFIGURATION_FILE, :modify) do
@logger.info(self) do
"#{CONFIGURATION_FILE} has been just updated, exiting.."
end
# We assume here that the service will be restarted by
# the service manager.
exit 0
end
inotify
rescue Errno::ENOENT => e
@logger.error(self) do
e.message
end
# We assume here that the service will be restarted by
# the service manager.
exit e.class::Errno
end
def setup_service_addr
# NOTE: We need the service_addr to be defined on one of the interfaces
# inside the host, one natural choice is the loopback interface (lo).
# Effectively we set it once, subsequent restarts of the service should
# honor the idempotence.
ip_address_add_cmd = lambda do |cidr_host, nic_device|
check = "[ -n \"$(ip a s to '#{cidr_host}' dev '#{nic_device}')\" ]"
apply = "ip a a '#{cidr_host}' dev '#{nic_device}'"
"#{check.strip} >/dev/null 2>&1 || #{apply.strip}"
end
system ip_address_add_cmd.call "#{@service_addr}/32", 'lo'
end
def setup_socket(listen = Socket::SOMAXCONN)
# NOTE: Must be executed before calling bind(), otherwise it fails..
setup_service_addr
sock = Socket.new Socket::AF_INET, Socket::SOCK_STREAM, 0
sock.setsockopt Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1
@logger.info(self) do
"Bind #{Addrinfo.tcp(@service_addr, @onegate_port).inspect}"
end
sock.bind Socket.pack_sockaddr_in(@onegate_port, @service_addr)
sock.listen listen
sock
end
def glue_streams(stream1, stream2, task)
Async do
concurrent = []
concurrent << task.async do
while (chunk = stream1.read_partial)
stream2.write chunk
stream2.flush
end
end
concurrent << task.async do
while (chunk = stream2.read_partial)
stream1.write chunk
stream1.flush
end
end
concurrent.each(&:wait)
end
end
def glue_peers(task)
@proxy_ep.accept do |vm_peer|
@logger.debug(self) do
"Accept #{vm_peer.remote_address.inspect}"
end
begin
gate_ep = Async::IO::Endpoint.tcp @onegate_addr,
@onegate_port
gate_ep.connect do |gate_peer|
vm_stream, gate_stream = Async::IO::Stream.new(vm_peer),
Async::IO::Stream.new(gate_peer)
glue_streams(vm_stream, gate_stream, task).wait
@logger.debug(self) do
"Close #{gate_peer.remote_address.inspect}"
end
gate_peer.close
end
rescue Errno::ECONNREFUSED,
Errno::ECONNRESET,
Errno::EHOSTUNREACH,
Errno::ETIMEDOUT => e
@logger.error(self) do
e.message
end
end
@logger.debug(self) do
"Close #{vm_peer.remote_address.inspect}"
end
vm_peer.close
end
end
end
if caller.empty?
options = DEFAULT_OPTIONS.dup
# NOTE: The "CONFIGURATION_FILE" is updated during the host sync procedure.
begin
options.merge! YAML.load_file(CONFIGURATION_FILE)
rescue StandardError => e
warn "Error parsing config file #{CONFIGURATION_FILE}: #{e.message}"
exit 1
end
puts <<~HEADER
--------------------------------------
Proxy configuration
--------------------------------------
#{options.pretty_inspect.strip}
--------------------------------------
HEADER
service = OneGateProxy.new options
service.run
end

49
src/vnm_mad/remotes/OpenNebulaNetwork.conf
View File

@ -97,3 +97,52 @@
# :ip_link_conf:
# :udp6zerocsumrx:
# :tos: 3
################################################################################
# TProxy / OneGate Options
################################################################################
# Each entry in the :tproxy array is a definition of a proxy instance:
#
# Link-local address and port used by VM guests to access the proxy.
# :service_addr: 169.254.16.9
# :service_port: 5030
#
# IP address and port of a service to proxy connections to.
# It must be accessible from the hypervisor hosts.
# :remote_addr: 10.11.12.13
# :remote_port: 5030
#
# Optionally a VNET can be specified by name or id.
# If it's not defined (the default behavior), then the proxy code will be
# applied to all VNETs / NICs of all involved VM guests.
# :network: vnet
# :network_id: 0
#
# The simplest example of an OneGate proxy config applied to all VNETs:
# :tproxy:
# - :service_addr: 169.254.16.9
# :service_port: 5030
# :remote_addr: 10.11.12.13 # HA VIP
# :remote_port: 5030
# Default log level for the proxy process.
# Logs are located on hypervisor hosts at /var/log/one_tproxy.log.
# The proxy process can be restarted on demand (as root): `/var/tmp/one/vnm/tproxy restart`.
#:tproxy_debug_level: 2 # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
#:tproxy:
#- :service_addr: 169.254.16.9
# :service_port: 5030
# :remote_addr: 10.11.12.13
# :remote_port: 5030
#- :network: vnet0
# :service_addr: 169.254.16.9
# :service_port: 4321
# :remote_addr: 10.11.12.13
# :remote_port: 1234
#- :network_id: 0
# :service_addr: 169.254.16.9
# :service_port: 4321
# :remote_addr: 10.11.12.13
# :remote_port: 1234

39
src/vnm_mad/remotes/lib/command.rb
View File

@ -35,7 +35,10 @@ module VNMMAD
:ovs_ofctl => 'sudo -n ovs-ofctl',
:ovs_appctl => 'sudo -n ovs-appctl',
:lsmod => 'lsmod',
:ipset => 'sudo -n ipset'
:ipset => 'sudo -n ipset',
:nft => 'sudo -n nft',
:sysctl => 'sudo -n sysctl',
:tproxy => 'sudo -n /var/tmp/one/vnm/tproxy'
}
# Adjust :ip[6]tables commands to work with legacy versions
@ -129,6 +132,40 @@ module VNMMAD
Open3.capture3(cmd_str)
end
# Executes a command (paranoid version)
# @return [String, String, Process::Status] the standard output,
# standard error and
# status returned by
# Open3.capture3
def self.no_shell(sym, *args, **opts)
terminate = (v = opts.delete(:term)).nil? ? true : v
args = COMMANDS[sym].split(' ') + args
o, e, s = Open3.capture3(*args, **opts)
if s.success?
OpenNebula.log_info "Executed \"#{args.join(' ')}\"."
OpenNebula.log_info Base64.strict_encode64(opts[:stdin_data]) \
unless opts[:stdin_data].nil?
else
if terminate
OpenNebula.log_error "Command \"#{args.join(' ')}\" failed."
OpenNebula.log_error Base64.strict_encode64(opts[:stdin_data]) \
unless opts[:stdin_data].nil?
OpenNebula.log_error e
exit(s.exitstatus)
else
OpenNebula.log_error "Command \"#{args.join(' ')}\" failed (recovered)."
OpenNebula.log_error Base64.strict_encode64(opts[:stdin_data]) \
unless opts[:stdin_data].nil?
OpenNebula.log_error e
end
end
[o, e, s]
end
end
end

17
src/vnm_mad/remotes/lib/no_vlan.rb
View File

@ -17,7 +17,7 @@
module VNMMAD
# NoVLANDriver class
class NoVLANDriver< VNMMAD::VLANDriver
class NoVLANDriver < VNMMAD::VLANDriver
# Attributes that can be updated on update_nic action
SUPPORTED_UPDATE = [
@ -42,7 +42,10 @@ module VNMMAD
# Create the bridge.
create_bridge(@nic)
# Return if vlan device is already in the bridge.
# Setup transparent proxies.
TProxy.setup_tproxy(@nic, :up)
# Skip if vlan device is already in the bridge.
next if !@nic[:phydev] || @nic[:phydev].empty? ||
@bridges[@nic[:bridge]].include?(@nic[:phydev])
@ -77,26 +80,28 @@ module VNMMAD
next if @bridges[@nic[:bridge]].nil?
# Return if the bridge doesn't exist because it was already
# Skip if the bridge doesn't exist because it was already
# deleted (handles last vm with multiple nics on the same
# vlan)
next unless @bridges.include? @nic[:bridge]
# Return if we want to keep the empty bridge
# Skip if we want to keep the empty bridge
next if @nic[:conf][:keep_empty_bridge]
# Return if the phydev device is not the only left device in
# Skip if the phydev device is not the only left device in
# the bridge.A
if @nic[:phydev].nil?
keep = !@bridges[@nic[:bridge]].empty?
else
keep = @bridges[@nic[:bridge]].length > 1 ||
!@bridges[@nic[:bridge]].include?(@nic[:phydev])
end
next if keep
# Setup transparent proxies.
TProxy.setup_tproxy(@nic, :down)
# Delete the bridge.
OpenNebula.exec_and_log("#{command(:ip)} link delete #{@nic[:bridge]}")

272
src/vnm_mad/remotes/lib/tproxy Executable file
View File

@ -0,0 +1,272 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
# frozen_string_literal: true
# rubocop:disable Lint/MissingCopEnableDirective
# rubocop:disable Metrics/ParameterLists
# rubocop:disable Style/Documentation
# rubocop:disable Style/GlobalVars
# rubocop:disable Style/ParallelAssignment
RUBY_LIB_LOCATION = '/usr/lib/one/ruby'
GEMS_LOCATION = '/usr/share/one/gems'
LOG_LOCATION = '/var/log'
RUN_LOCATION = '/var/run'
REMOTES_LOCATION = '/var/tmp/one'
CONFIGURATION_FILE = REMOTES_LOCATION + '/etc/vnm/OpenNebulaNetwork.conf'
# %%RUBYGEMS_SETUP_BEGIN%%
if File.directory?(GEMS_LOCATION)
real_gems_path = File.realpath(GEMS_LOCATION)
if !defined?(Gem) || Gem.path != [real_gems_path]
$LOAD_PATH.reject! {|p| p =~ /vendor_ruby/ }
# Suppress warnings from Rubygems
# https://github.com/OpenNebula/one/issues/5379
begin
verb = $VERBOSE
$VERBOSE = nil
require 'rubygems'
Gem.use_paths(real_gems_path)
ensure
$VERBOSE = verb
end
end
end
# %%RUBYGEMS_SETUP_END%%
$LOAD_PATH << RUBY_LIB_LOCATION
require 'async/io'
require 'async/io/stream'
require 'async/io/trap'
require 'console'
require 'daemons'
require 'json'
require 'open3'
require 'socket'
require 'yaml'
DEFAULT_CONFIG = {
:tproxy_debug_level => 2, # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
:app_name => 'one_tproxy'
}.freeze
LOG_LEVEL_MAP = {
0 => 3, # ERROR
1 => 2, # WARN
2 => 1, # INFO
3 => 0 # DEBUG
}.freeze
$config = DEFAULT_CONFIG.dup
$logger = nil
module VNMMAD
module TProxy
# A single async TCP transparent proxy implementation, it binds to a single port and
# marks outgoing packets with SO_MARK.
class Single
def initialize(bport, daddr, dport, smark)
@bport, @daddr, @dport, @smark = bport, daddr, dport, smark
@socket = setup_socket('127.0.0.1', @bport, @smark)
@proxy_ep = Async::IO::Endpoint.socket(@socket)
end
def params
[@bport, @daddr, @dport, @smark]
end
def run
Async do |task|
glue_peers(@task = task)
end
end
def stop
$logger.info(self) do
"Stop #{Addrinfo.tcp('127.0.0.1', @bport).inspect}"
end
@socket.close
@task.stop
end
private
def setup_socket(baddr, bport, smark, listen = Socket::SOMAXCONN)
sock = Socket.new Socket::AF_INET, Socket::SOCK_STREAM, 0
sock.setsockopt Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1
sock.setsockopt Socket::SOL_SOCKET, Socket::SO_MARK, smark
sock.setsockopt Socket::SOL_IP, Socket::IP_TRANSPARENT, 1
$logger.info(self) do
"Bind #{Addrinfo.tcp(baddr, bport).inspect}"
end
sock.bind Socket.pack_sockaddr_in(bport, baddr)
sock.listen listen
sock
end
def glue_streams(stream1, stream2, task)
task.async do |subtask|
concurrent = []
concurrent << subtask.async do
while (chunk = stream1.read_partial)
stream2.write chunk
stream2.flush
end
end
concurrent << subtask.async do
while (chunk = stream2.read_partial)
stream1.write chunk
stream1.flush
end
end
concurrent.each(&:wait)
end
end
def glue_peers(task)
@proxy_ep.accept do |client_peer|
$logger.debug(self) do
"Accept #{client_peer.remote_address.inspect}"
end
begin
server_ep = Async::IO::Endpoint.tcp @daddr,
@dport
server_ep.connect do |server_peer|
client_stream, server_stream = Async::IO::Stream.new(client_peer),
Async::IO::Stream.new(server_peer)
glue_streams(client_stream, server_stream, task).wait
ensure
$logger.debug(self) do
"Close #{server_peer.remote_address.inspect}"
end
server_peer.close
end
rescue Errno::ECONNREFUSED,
Errno::ECONNRESET,
Errno::EHOSTUNREACH,
Errno::ETIMEDOUT => e
$logger.error(self) do
e.message
end
end
ensure
$logger.debug(self) do
"Close #{client_peer.remote_address.inspect}"
end
client_peer.close
end
end
end
# Combine multiple proxies into one async service.
class Multi
def initialize
@single = {}
@sighup = Async::IO::Trap.new :HUP
@sighup.install!
end
def run
Async do
reload
@sighup.wait { reload }
end
end
private
def reload
cmd = "nft -j list map ip #{$config[:app_name]} proxies"
o, e, s = Open3.capture3(*cmd.split(' '))
unless s.success?
$logger.error(self) do
"Error getting proxy map from nftables: #{e}"
end
return
end
proxies = JSON.parse(o)
&.dig('nftables')
&.find {|item| !item['map'].nil? }
&.dig('map', 'elem')
.to_h {|item| item.map(&:values).map(&:flatten) }
# Stop and remove cancelled proxies.
@single.each do |k, v|
next if proxies.key?(k) && v.params == proxies[k]
@single.delete(k)&.stop
end
# Create and start missing proxies.
proxies.each do |k, v|
next if @single.key?(k)
(@single[k] = Single.new(*v)).run
rescue StandardError => e
$logger.error(self) do
e.message
end
end
end
end
end
end
Daemons.run_proc($config[:app_name], :dir => RUN_LOCATION, :log_dir => LOG_LOCATION) do
CustomLogger = Console::Filter[:debug => 0, :info => 1, :warn => 2, :error => 3]
logfile = File.open "#{LOG_LOCATION}/#{$config[:app_name]}.log", 'a'
logfile.sync = true
serialized = Console::Serialized::Logger.new logfile
$logger = CustomLogger.new serialized, :level => 0
# The "CONFIGURATION_FILE" is updated during the host sync procedure.
begin
$config.merge! YAML.load_file(CONFIGURATION_FILE)
rescue StandardError => e
$logger.error(self) do
"Error parsing config file #{CONFIGURATION_FILE}: #{e.message}"
end
exit(-1)
end
$logger = CustomLogger.new serialized, :level => LOG_LEVEL_MAP[$config[:tproxy_debug_level]]
VNMMAD::TProxy::Multi.new.run
end

368
src/vnm_mad/remotes/lib/tproxy.rb Normal file
View File

@ -0,0 +1,368 @@
# -------------------------------------------------------------------------- #
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
require 'base64'
require 'erb'
require 'json'
require 'open3'
require 'resolv'
module VNMMAD
# Module to handle transparent proxies.
module TProxy
# A helper class to manage the tproxy mapping in nftables.
class ProxyMap
def initialize(port_range = (16_000...32_000))
@range = port_range
reload
end
def reload
@map = self.class.load_proxy_map
@ports = self.class.load_proxy_ports(@map)
end
def get_port(bridge, service_addr, service_port)
if (v = @map[[bridge, service_addr, service_port]]).nil?
next_port
else
v[0]
end
end
# Delete all map entries that do not match current config (for a specific bridge).
# This is a best-effort type of operation.
def cleanup(bridge, keep: @ports)
to_delete = @map.each_with_object({}) do |(k, v), acc|
acc[k] = v if k[0] == bridge && !keep.include?(v[0])
end
TProxy.nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding), :term => false)
<%- to_delete.each do |k, v| -%>
delete element ip one_tproxy proxies { "<%= k[0] %>" . <%= k[1..(-1)].join(' . ') %> }
<%- end -%>
NFT
end
def self.load_proxy_map
TProxy.nft_json('list map ip one_tproxy proxies', :term => false)
.dig(0, 'nftables')
&.find {|item| !item['map'].nil? }
&.dig('map', 'elem')
.to_h {|item| item.map(&:values).map(&:flatten) }
end
def self.load_proxy_ports(proxy_map = get_proxy_map)
proxy_map.map {|_, v| v[0] }
end
private
def next_port
# This is not the most efficient implementation, but we can safely assume
# it is likely the number of proxy ports openned should be low.
if (v = @range.find {|port| !@ports.include?(port) }).nil?
nil
else
@ports << v
v
end
end
end
# The entry point for the tproxy feature.
def self.setup_tproxy(nic, direction)
# Short-circuit if no tproxy config is provided.
return if CONF[:tproxy].to_a.empty?
begin
nonce = Integer(nic[:network_id]) % 16_000
rescue ArgumentError
return
end
proxy_map = ProxyMap.new
proxy_opts = CONF[:tproxy].to_a.each_with_object([]) do |conf, acc|
next if !conf[:network_id].nil? \
&& Integer(conf[:network_id]) != Integer(nic[:network_id])
next if !conf[:network].nil? \
&& conf[:network] != nic[:network]
next if conf[:remote_addr].nil? || conf[:remote_port].nil?
opts = {
# Multiple proxies in the same VNET can reuse the same routing tables and rules.
:rt_in => 16_000 + nonce,
:mark_in => 16_000 + nonce,
:rt_out => 32_000 + nonce,
:mark_out => 32_000 + nonce,
:service_addr => conf[:service_addr] || '169.254.16.9',
:service_port => Integer(conf[:service_port] || conf[:remote_port]),
:remote_addr => conf[:remote_addr],
:remote_port => Integer(conf[:remote_port])
}
next unless opts[:service_addr] =~ Resolv::IPv4::Regex
next unless opts[:remote_addr] =~ Resolv::IPv4::Regex
opts[:proxy_port] = proxy_map.get_port(nic[:bridge],
opts[:service_addr],
opts[:service_port])
acc << opts
rescue ArgumentError
next
end
# Short-circuit if no valid config is recognized.
return if proxy_opts.empty?
if direction == :up
proxy_map.cleanup(nic[:bridge],
:keep => proxy_opts.map {|opts| opts[:proxy_port] })
enable_tproxy(nic, proxy_opts)
run_tproxy('start')
run_tproxy('reload')
else
run_tproxy('stop')
disable_tproxy(nic, proxy_opts)
proxy_map.cleanup(nic[:bridge], :keep => []) # delete all
end
end
def self.enable_tproxy(nic, proxy_opts)
br = nic[:bridge]
# Get the MAC of the bridge to later use it in NFT rules.
mac = ip_json(<<~LINK).dig(0, 0, 'address')
link show dev #{br}
LINK
return if mac.nil?
# Completely disable reverse path filtering for the bridge at hand. This is required
# for both proxy-arp to work and routing the response packets back to VM guests.
sysctl(<<~SYSCTL)
net.ipv4.conf.all.rp_filter=0
SYSCTL
sysctl(<<~SYSCTL)
net.ipv4.conf.#{br}.rp_filter=0
SYSCTL
proxy_opts.each do |opts|
# Enable proxy-arp.
ip_neighbour_add_proxy(<<~NEIGHBOUR)
#{opts[:service_addr]} dev #{br}
NEIGHBOUR
# This is required for proxy-arp to work.
ip_route_replace(<<~ROUTE)
#{opts[:service_addr]} dev lo
ROUTE
ip_rule_add(<<~RULE)
fwmark #{opts[:mark_in]} lookup #{opts[:rt_in]}
RULE
ip_route_replace(<<~ROUTE)
local default dev lo table #{opts[:rt_in]}
ROUTE
ip_rule_add(<<~RULE)
fwmark #{opts[:mark_out]} lookup #{opts[:rt_out]}
RULE
# Inject response packets (marked with :mark_out) back into the bridge.
ip_route_replace(<<~ROUTE)
default dev #{br} table #{opts[:rt_out]}
ROUTE
end
# This nftables config can be considered to be a failsafe. In case where proxy-arp
# does not work for any reason, users can still manually set permanent ARP mapping
# `arp -s 169.254.16.9 00:00:00:00:00:00` inside VM guests. Setting the mapping avoids
# ARP resolution, then nftables overrides / corrects the destination MAC address.
nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding))
table bridge one_tproxy {
chain #{br} {
type filter hook prerouting priority dstnat; policy accept;
}
}
flush chain bridge one_tproxy #{br}
table bridge one_tproxy {
chain #{br} {
<%- proxy_opts.each do |opts| -%>
meta ibrname "#{br}" \\
ip daddr <%= opts[:service_addr] %> \\
tcp dport <%= opts[:service_port] %> \\
counter \\
meta pkttype set host ether daddr set #{mac} \\
accept
<%- end -%>
}
}
NFT
# The tproxy.rb process reads its config from the "ip one_tproxy proxies" map
# defined in nftables, that way users can manually restart the proxy on demand
# without the need of providing any command line arguments. The map is managed
# by the driver, proxy only reads its contents.
nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding))
table ip one_tproxy {
map proxies {
type ifname . ipv4_addr . inet_service : inet_service . ipv4_addr . inet_service . mark;
elements = {
<%- proxy_opts.each do |opts| -%>
"#{br}" . <%= opts[:service_addr] %> . <%= opts[:service_port] %> \\
: <%= opts[:proxy_port] %> . <%= opts[:remote_addr] %> . <%= opts[:remote_port] %> . <%= opts[:mark_out] %>,
<%- end -%>
}
}
chain #{br} {
type filter hook prerouting priority mangle; policy accept;
}
}
flush chain ip one_tproxy #{br}
table ip one_tproxy {
chain #{br} {
<%- proxy_opts.each do |opts| -%>
iifname "#{br}" \\
meta l4proto tcp \\
ip daddr <%= opts[:service_addr] %> \\
tcp dport <%= opts[:service_port] %> \\
counter \\
mark set <%= opts[:mark_in] %> \\
tproxy to 127.0.0.1:<%= opts[:proxy_port] %>
<%- end -%>
}
}
NFT
end
def self.disable_tproxy(nic, proxy_opts)
br = nic[:bridge]
proxy_opts.each do |opts|
ip_rule_del(<<~RULE)
fwmark #{opts[:mark_in]} lookup #{opts[:rt_in]}
RULE
ip_rule_del(<<~RULE)
fwmark #{opts[:mark_out]} lookup #{opts[:rt_out]}
RULE
end
nft(<<~NFT)
table bridge one_tproxy {
chain #{br} {
type filter hook prerouting priority dstnat; policy accept;
}
}
flush chain bridge one_tproxy #{br}
NFT
nft(<<~NFT)
table ip one_tproxy {
chain #{br} {
type filter hook prerouting priority mangle; policy accept;
}
}
flush chain ip one_tproxy #{br}
NFT
end
def self.run_tproxy(cmd)
run(:tproxy, *cmd.strip.split(' '))
end
def self.ip_json(cmd, **opts)
o, e, s = run(:ip_unpriv, '-j', *cmd.strip.split(' '), **opts)
if s.success?
[JSON.parse(o), e, s]
else
[{}, e, s]
end
end
def self.ip_neighbour_add_proxy(cmd)
run(:ip, 'neighbour', 'add', 'proxy', *cmd.strip.split(' '))
end
def self.ip_rule_add(cmd)
args = cmd.strip.split(' ')
if (check = run(:ip, 'rule', 'list', *args))[0].strip.empty?
run(:ip, 'rule', 'add', *args)
else
check
end
end
def self.ip_rule_del(cmd)
args = cmd.strip.split(' ')
if (check = run(:ip, 'rule', 'list', *args))[0].strip.empty?
check
else
run(:ip, 'rule', 'del', *args)
end
end
def self.ip_route_replace(cmd)
run(:ip, 'route', 'replace', *cmd.strip.split(' '))
end
def self.nft_json(cmd, **opts)
o, e, s = run(:nft, '-j', *cmd.strip.split(' '), **opts)
if s.success?
[JSON.parse(o), e, s]
else
[{}, e, s]
end
end
def self.nft(script, **opts)
run(:nft, '-f-', **opts, :stdin_data => script)
end
def self.sysctl(cmd)
run(:sysctl, *cmd.strip.split(' '))
end
private_class_method def self.run(sym, *args, **opts)
VNMNetwork::Command.no_shell(sym, *args, **opts)
end
end
end

15
src/vnm_mad/remotes/lib/vlan.rb
View File

@ -53,10 +53,13 @@ module VNMMAD
# Create the bridge.
create_bridge(@nic)
# Setup transparent proxies.
TProxy.setup_tproxy(@nic, :up)
# Check that no other vlans are connected to this bridge
validate_vlan_id if @nic[:conf][:validate_vlan_id]
# Return if vlan device is already in the bridge.
# Skip if vlan device is already in the bridge.
next if @bridges[@nic[:bridge]].include? @nic[:vlan_dev]
# Create vlan device.
@ -113,15 +116,15 @@ module VNMMAD
# Get the name of the vlan device.
gen_vlan_dev_name
# Return if the bridge doesn't exist because it was already
# Skip if the bridge doesn't exist because it was already
# deleted (handles last vm with multiple nics on the same
# vlan)
next unless @bridges.include? @nic[:bridge]
# Return if we want to keep the empty bridge
# Skip if we want to keep the empty bridge
next if @nic[:conf][:keep_empty_bridge]
# Return if the vlan device is not the only left device in
# Skip if the vlan device is not the only left device in
# the bridge.
next if (@bridges[@nic[:bridge]].length > 1) || \
!@bridges[@nic[:bridge]].include?(@nic[:vlan_dev])
@ -131,9 +134,13 @@ module VNMMAD
@bridges[@nic[:bridge]].delete(@nic[:vlan_dev])
# Setup transparent proxies.
TProxy.setup_tproxy(@nic, :down)
# Delete the bridge.
OpenNebula.exec_and_log("#{command(:ip)} link delete"\
" #{@nic[:bridge]}")
@bridges.delete(@nic[:bridge])
end
end

16
src/vnm_mad/remotes/lib/vnm_driver.rb
View File

@ -14,8 +14,10 @@
# limitations under the License. #
#--------------------------------------------------------------------------- #
require 'shellwords'
require 'json'
require 'open3'
require 'resolv'
require 'shellwords'
require_relative 'vf'
@ -224,12 +226,18 @@ module VNMMAD
return if @bridges.key?(nic[:bridge])
OpenNebula.exec_and_log("#{command(:ip)} link add name " \
"#{nic[:bridge]} type bridge #{list_bridge_options(nic)}", nil, 2)
"'#{nic[:bridge]}' type bridge #{list_bridge_options(nic)}", nil, 2)
@bridges[nic[:bridge]] = []
ensure
# NOTE: We read here the randomized MAC address that the bridge is created with, then
# we set the same MAC address back again. This prevents automatic change of the
# bridge MAC address when new interfaces are inserted into the bridge.
mac = JSON.parse(`#{command(:ip_unpriv)} -j \
link show dev '#{nic[:bridge]}'`).dig(0, 'address')
OpenNebula.exec_and_log("#{command(:ip)} " \
"link set #{nic[:bridge]} up")
"link set dev '#{nic[:bridge]}' address #{mac} up")
end
# Reads config and return str with switches
@ -281,8 +289,6 @@ module VNMMAD
bridge_options_str.strip
end
# Returns a filter object based on the contents of the template
#
# @return SGDriver object

18
src/vnm_mad/remotes/lib/vnmmad.rb
View File

@ -33,26 +33,30 @@ require 'sg_driver'
require 'vlan'
require 'no_vlan'
require 'scripts_common'
require 'tproxy'
Dir[File.expand_path('vnmmad-load.d', File.dirname(__FILE__)) + "/*.rb"].each{ |f| require f }
include OpenNebula
begin
NAME = File.join(File.dirname(__FILE__), "../etc/vnm/OpenNebulaNetwork.conf")
NAME = File.join(File.dirname(__FILE__), '../etc/vnm/OpenNebulaNetwork.conf')
CONF = YAML.load_file(NAME)
rescue
# Default configuration values
CONF = {
:arp_cache_poisoning => true,
:vxlan_mc => "239.0.0.0",
:vxlan_ttl => "16",
:vxlan_mtu => "1500",
:vxlan_mc => '239.0.0.0',
:vxlan_ttl => '16',
:vxlan_mtu => '1500',
:validate_vlan_id => false,
:vlan_mtu => "1500",
:ipset_maxelem => "65536",
:vlan_mtu => '1500',
:ipset_maxelem => '65536',
:keep_empty_bridge => false,
:datastore_location => '/var/lib/one/datastores'
:datastore_location => '/var/lib/one/datastores',
:tproxy_debug_level => 2, # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
:tproxy_process_owner => 'oneadmin',
:tproxy => []
}
end