mirror of
https://github.com/OpenNebula/one.git
synced 2025-01-03 01:17:41 +03:00
M #-: Remove tproxy logic
This commit is contained in:
Ruben S. Montero
parent
b7d070459b
commit
f194c1241f
@ -1538,9 +1538,7 @@ NETWORK_FILES="src/vnm_mad/remotes/lib/vnm_driver.rb \
|
|||||||
src/vnm_mad/remotes/lib/no_vlan.rb \
|
src/vnm_mad/remotes/lib/no_vlan.rb \
|
||||||
src/vnm_mad/remotes/lib/security_groups.rb \
|
src/vnm_mad/remotes/lib/security_groups.rb \
|
||||||
src/vnm_mad/remotes/lib/security_groups_iptables.rb \
|
src/vnm_mad/remotes/lib/security_groups_iptables.rb \
|
||||||
src/vnm_mad/remotes/lib/nic.rb \
|
src/vnm_mad/remotes/lib/nic.rb"
|
||||||
src/vnm_mad/remotes/lib/tproxy \
|
|
||||||
src/vnm_mad/remotes/lib/tproxy.rb"
|
|
||||||
|
|
||||||
NETWORK_8021Q_FILES="src/vnm_mad/remotes/802.1Q/clean \
|
NETWORK_8021Q_FILES="src/vnm_mad/remotes/802.1Q/clean \
|
||||||
src/vnm_mad/remotes/802.1Q/post \
|
src/vnm_mad/remotes/802.1Q/post \
|
||||||
|
|||||||
@ -6,7 +6,7 @@ Cmnd_Alias ONE_HA = /usr/bin/systemctl start opennebula-flow, /usr/bin/systemctl
|
|||||||
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
|
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
|
||||||
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
|
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
|
||||||
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
|
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
|
||||||
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *, /usr/sbin/nft, /usr/sbin/sysctl net.ipv4.conf.*, /var/tmp/one/vnm/tproxy
|
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *
|
||||||
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||||
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
|
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
|
||||||
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu
|
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu
|
||||||
|
|||||||
@ -6,7 +6,7 @@ Cmnd_Alias ONE_HA = /usr/bin/systemctl start opennebula-flow, /usr/bin/systemctl
|
|||||||
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
|
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
|
||||||
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
|
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
|
||||||
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
|
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
|
||||||
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *, /usr/sbin/nft, /usr/sbin/sysctl net.ipv4.conf.*, /var/tmp/one/vnm/tproxy
|
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *
|
||||||
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||||
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
|
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
|
||||||
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu
|
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu
|
||||||
|
|||||||
@ -33,10 +33,7 @@ class Sudoers
|
|||||||
'ip neighbour *',
|
'ip neighbour *',
|
||||||
'ip route *',
|
'ip route *',
|
||||||
'ip rule *',
|
'ip rule *',
|
||||||
'ip tuntap *',
|
'ip tuntap *'
|
||||||
'nft',
|
|
||||||
'sysctl net.ipv4.conf.*',
|
|
||||||
'/var/tmp/one/vnm/tproxy'
|
|
||||||
],
|
],
|
||||||
:LVM => [
|
:LVM => [
|
||||||
'lvcreate', 'lvremove', 'lvs', 'vgdisplay', 'lvchange', 'lvscan', 'lvextend'
|
'lvcreate', 'lvremove', 'lvs', 'vgdisplay', 'lvchange', 'lvscan', 'lvextend'
|
||||||
|
|||||||
@ -97,52 +97,3 @@
|
|||||||
# :ip_link_conf:
|
# :ip_link_conf:
|
||||||
# :udp6zerocsumrx:
|
# :udp6zerocsumrx:
|
||||||
# :tos: 3
|
# :tos: 3
|
||||||
|
|
||||||
################################################################################
|
|
||||||
# TProxy / OneGate Options
|
|
||||||
################################################################################
|
|
||||||
|
|
||||||
# Each entry in the :tproxy array is a definition of a proxy instance:
|
|
||||||
#
|
|
||||||
# Link-local address and port used by VM guests to access the proxy.
|
|
||||||
# :service_addr: 169.254.16.9
|
|
||||||
# :service_port: 5030
|
|
||||||
#
|
|
||||||
# IP address and port of a service to proxy connections to.
|
|
||||||
# It must be accessible from the hypervisor hosts.
|
|
||||||
# :remote_addr: 10.11.12.13
|
|
||||||
# :remote_port: 5030
|
|
||||||
#
|
|
||||||
# Optionally a VNET can be specified by name or id.
|
|
||||||
# If it's not defined (the default behavior), then the proxy code will be
|
|
||||||
# applied to all VNETs / NICs of all involved VM guests.
|
|
||||||
# :network: vnet
|
|
||||||
# :network_id: 0
|
|
||||||
#
|
|
||||||
# The simplest example of an OneGate proxy config applied to all VNETs:
|
|
||||||
# :tproxy:
|
|
||||||
# - :service_addr: 169.254.16.9
|
|
||||||
# :service_port: 5030
|
|
||||||
# :remote_addr: 10.11.12.13 # HA VIP
|
|
||||||
# :remote_port: 5030
|
|
||||||
|
|
||||||
# Default log level for the proxy process.
|
|
||||||
# Logs are located on hypervisor hosts at /var/log/one_tproxy.log.
|
|
||||||
# The proxy process can be restarted on demand (as root): `/var/tmp/one/vnm/tproxy restart`.
|
|
||||||
#:tproxy_debug_level: 2 # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
|
|
||||||
|
|
||||||
#:tproxy:
|
|
||||||
#- :service_addr: 169.254.16.9
|
|
||||||
# :service_port: 5030
|
|
||||||
# :remote_addr: 10.11.12.13
|
|
||||||
# :remote_port: 5030
|
|
||||||
#- :network: vnet0
|
|
||||||
# :service_addr: 169.254.16.9
|
|
||||||
# :service_port: 4321
|
|
||||||
# :remote_addr: 10.11.12.13
|
|
||||||
# :remote_port: 1234
|
|
||||||
#- :network_id: 0
|
|
||||||
# :service_addr: 169.254.16.9
|
|
||||||
# :service_port: 4321
|
|
||||||
# :remote_addr: 10.11.12.13
|
|
||||||
# :remote_port: 1234
|
|
||||||
|
|||||||
@ -35,10 +35,7 @@ module VNMMAD
|
|||||||
:ovs_ofctl => 'sudo -n ovs-ofctl',
|
:ovs_ofctl => 'sudo -n ovs-ofctl',
|
||||||
:ovs_appctl => 'sudo -n ovs-appctl',
|
:ovs_appctl => 'sudo -n ovs-appctl',
|
||||||
:lsmod => 'lsmod',
|
:lsmod => 'lsmod',
|
||||||
:ipset => 'sudo -n ipset',
|
:ipset => 'sudo -n ipset'
|
||||||
:nft => 'sudo -n nft',
|
|
||||||
:sysctl => 'sudo -n sysctl',
|
|
||||||
:tproxy => 'sudo -n /var/tmp/one/vnm/tproxy'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Adjust :ip[6]tables commands to work with legacy versions
|
# Adjust :ip[6]tables commands to work with legacy versions
|
||||||
|
|||||||
@ -42,9 +42,6 @@ module VNMMAD
|
|||||||
# Create the bridge.
|
# Create the bridge.
|
||||||
create_bridge(@nic)
|
create_bridge(@nic)
|
||||||
|
|
||||||
# Setup transparent proxies.
|
|
||||||
TProxy.setup_tproxy(@nic, :up)
|
|
||||||
|
|
||||||
# Skip if vlan device is already in the bridge.
|
# Skip if vlan device is already in the bridge.
|
||||||
next if !@nic[:phydev] || @nic[:phydev].empty? ||
|
next if !@nic[:phydev] || @nic[:phydev].empty? ||
|
||||||
@bridges[@nic[:bridge]].include?(@nic[:phydev])
|
@bridges[@nic[:bridge]].include?(@nic[:phydev])
|
||||||
@ -99,9 +96,6 @@ module VNMMAD
|
|||||||
|
|
||||||
next if keep
|
next if keep
|
||||||
|
|
||||||
# Setup transparent proxies.
|
|
||||||
TProxy.setup_tproxy(@nic, :down)
|
|
||||||
|
|
||||||
# Delete the bridge.
|
# Delete the bridge.
|
||||||
OpenNebula.exec_and_log("#{command(:ip)} link delete #{@nic[:bridge]}")
|
OpenNebula.exec_and_log("#{command(:ip)} link delete #{@nic[:bridge]}")
|
||||||
|
|
||||||
|
|||||||
@ -1,272 +0,0 @@
|
|||||||
#!/usr/bin/env ruby
|
|
||||||
# -------------------------------------------------------------------------- #
|
|
||||||
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
|
|
||||||
# #
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
|
||||||
# not use this file except in compliance with the License. You may obtain #
|
|
||||||
# a copy of the License at #
|
|
||||||
# #
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 #
|
|
||||||
# #
|
|
||||||
# Unless required by applicable law or agreed to in writing, software #
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, #
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
|
||||||
# See the License for the specific language governing permissions and #
|
|
||||||
# limitations under the License. #
|
|
||||||
#--------------------------------------------------------------------------- #
|
|
||||||
|
|
||||||
# frozen_string_literal: true
|
|
||||||
|
|
||||||
# rubocop:disable Lint/MissingCopEnableDirective
|
|
||||||
# rubocop:disable Metrics/ParameterLists
|
|
||||||
# rubocop:disable Style/Documentation
|
|
||||||
# rubocop:disable Style/GlobalVars
|
|
||||||
# rubocop:disable Style/ParallelAssignment
|
|
||||||
|
|
||||||
RUBY_LIB_LOCATION = '/usr/lib/one/ruby'
|
|
||||||
GEMS_LOCATION = '/usr/share/one/gems'
|
|
||||||
LOG_LOCATION = '/var/log'
|
|
||||||
RUN_LOCATION = '/var/run'
|
|
||||||
REMOTES_LOCATION = '/var/tmp/one'
|
|
||||||
CONFIGURATION_FILE = REMOTES_LOCATION + '/etc/vnm/OpenNebulaNetwork.conf'
|
|
||||||
|
|
||||||
# %%RUBYGEMS_SETUP_BEGIN%%
|
|
||||||
if File.directory?(GEMS_LOCATION)
|
|
||||||
real_gems_path = File.realpath(GEMS_LOCATION)
|
|
||||||
if !defined?(Gem) || Gem.path != [real_gems_path]
|
|
||||||
$LOAD_PATH.reject! {|p| p =~ /vendor_ruby/ }
|
|
||||||
|
|
||||||
# Suppress warnings from Rubygems
|
|
||||||
# https://github.com/OpenNebula/one/issues/5379
|
|
||||||
begin
|
|
||||||
verb = $VERBOSE
|
|
||||||
$VERBOSE = nil
|
|
||||||
require 'rubygems'
|
|
||||||
Gem.use_paths(real_gems_path)
|
|
||||||
ensure
|
|
||||||
$VERBOSE = verb
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
# %%RUBYGEMS_SETUP_END%%
|
|
||||||
|
|
||||||
$LOAD_PATH << RUBY_LIB_LOCATION
|
|
||||||
|
|
||||||
require 'async/io'
|
|
||||||
require 'async/io/stream'
|
|
||||||
require 'async/io/trap'
|
|
||||||
require 'console'
|
|
||||||
require 'daemons'
|
|
||||||
require 'json'
|
|
||||||
require 'open3'
|
|
||||||
require 'socket'
|
|
||||||
require 'yaml'
|
|
||||||
|
|
||||||
DEFAULT_CONFIG = {
|
|
||||||
:tproxy_debug_level => 2, # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
|
|
||||||
:app_name => 'one_tproxy'
|
|
||||||
}.freeze
|
|
||||||
|
|
||||||
LOG_LEVEL_MAP = {
|
|
||||||
0 => 3, # ERROR
|
|
||||||
1 => 2, # WARN
|
|
||||||
2 => 1, # INFO
|
|
||||||
3 => 0 # DEBUG
|
|
||||||
}.freeze
|
|
||||||
|
|
||||||
$config = DEFAULT_CONFIG.dup
|
|
||||||
$logger = nil
|
|
||||||
|
|
||||||
module VNMMAD
|
|
||||||
|
|
||||||
module TProxy
|
|
||||||
|
|
||||||
# A single async TCP transparent proxy implementation, it binds to a single port and
|
|
||||||
# marks outgoing packets with SO_MARK.
|
|
||||||
class Single
|
|
||||||
|
|
||||||
def initialize(bport, daddr, dport, smark)
|
|
||||||
@bport, @daddr, @dport, @smark = bport, daddr, dport, smark
|
|
||||||
|
|
||||||
@socket = setup_socket('127.0.0.1', @bport, @smark)
|
|
||||||
@proxy_ep = Async::IO::Endpoint.socket(@socket)
|
|
||||||
end
|
|
||||||
|
|
||||||
def params
|
|
||||||
[@bport, @daddr, @dport, @smark]
|
|
||||||
end
|
|
||||||
|
|
||||||
def run
|
|
||||||
Async do |task|
|
|
||||||
glue_peers(@task = task)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def stop
|
|
||||||
$logger.info(self) do
|
|
||||||
"Stop #{Addrinfo.tcp('127.0.0.1', @bport).inspect}"
|
|
||||||
end
|
|
||||||
@socket.close
|
|
||||||
@task.stop
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def setup_socket(baddr, bport, smark, listen = Socket::SOMAXCONN)
|
|
||||||
sock = Socket.new Socket::AF_INET, Socket::SOCK_STREAM, 0
|
|
||||||
|
|
||||||
sock.setsockopt Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1
|
|
||||||
sock.setsockopt Socket::SOL_SOCKET, Socket::SO_MARK, smark
|
|
||||||
|
|
||||||
sock.setsockopt Socket::SOL_IP, Socket::IP_TRANSPARENT, 1
|
|
||||||
|
|
||||||
$logger.info(self) do
|
|
||||||
"Bind #{Addrinfo.tcp(baddr, bport).inspect}"
|
|
||||||
end
|
|
||||||
|
|
||||||
sock.bind Socket.pack_sockaddr_in(bport, baddr)
|
|
||||||
sock.listen listen
|
|
||||||
sock
|
|
||||||
end
|
|
||||||
|
|
||||||
def glue_streams(stream1, stream2, task)
|
|
||||||
task.async do |subtask|
|
|
||||||
concurrent = []
|
|
||||||
concurrent << subtask.async do
|
|
||||||
while (chunk = stream1.read_partial)
|
|
||||||
stream2.write chunk
|
|
||||||
stream2.flush
|
|
||||||
end
|
|
||||||
end
|
|
||||||
concurrent << subtask.async do
|
|
||||||
while (chunk = stream2.read_partial)
|
|
||||||
stream1.write chunk
|
|
||||||
stream1.flush
|
|
||||||
end
|
|
||||||
end
|
|
||||||
concurrent.each(&:wait)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def glue_peers(task)
|
|
||||||
@proxy_ep.accept do |client_peer|
|
|
||||||
$logger.debug(self) do
|
|
||||||
"Accept #{client_peer.remote_address.inspect}"
|
|
||||||
end
|
|
||||||
|
|
||||||
begin
|
|
||||||
server_ep = Async::IO::Endpoint.tcp @daddr,
|
|
||||||
@dport
|
|
||||||
server_ep.connect do |server_peer|
|
|
||||||
client_stream, server_stream = Async::IO::Stream.new(client_peer),
|
|
||||||
Async::IO::Stream.new(server_peer)
|
|
||||||
|
|
||||||
glue_streams(client_stream, server_stream, task).wait
|
|
||||||
ensure
|
|
||||||
$logger.debug(self) do
|
|
||||||
"Close #{server_peer.remote_address.inspect}"
|
|
||||||
end
|
|
||||||
|
|
||||||
server_peer.close
|
|
||||||
end
|
|
||||||
rescue Errno::ECONNREFUSED,
|
|
||||||
Errno::ECONNRESET,
|
|
||||||
Errno::EHOSTUNREACH,
|
|
||||||
Errno::ETIMEDOUT => e
|
|
||||||
$logger.error(self) do
|
|
||||||
e.message
|
|
||||||
end
|
|
||||||
end
|
|
||||||
ensure
|
|
||||||
$logger.debug(self) do
|
|
||||||
"Close #{client_peer.remote_address.inspect}"
|
|
||||||
end
|
|
||||||
|
|
||||||
client_peer.close
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
# Combine multiple proxies into one async service.
|
|
||||||
class Multi
|
|
||||||
|
|
||||||
def initialize
|
|
||||||
@single = {}
|
|
||||||
@sighup = Async::IO::Trap.new :HUP
|
|
||||||
@sighup.install!
|
|
||||||
end
|
|
||||||
|
|
||||||
def run
|
|
||||||
Async do
|
|
||||||
reload
|
|
||||||
@sighup.wait { reload }
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def reload
|
|
||||||
cmd = "nft -j list map ip #{$config[:app_name]} proxies"
|
|
||||||
|
|
||||||
o, e, s = Open3.capture3(*cmd.split(' '))
|
|
||||||
|
|
||||||
unless s.success?
|
|
||||||
$logger.error(self) do
|
|
||||||
"Error getting proxy map from nftables: #{e}"
|
|
||||||
end
|
|
||||||
return
|
|
||||||
end
|
|
||||||
|
|
||||||
proxies = JSON.parse(o)
|
|
||||||
&.dig('nftables')
|
|
||||||
&.find {|item| !item['map'].nil? }
|
|
||||||
&.dig('map', 'elem')
|
|
||||||
.to_h {|item| item.map(&:values).map(&:flatten) }
|
|
||||||
|
|
||||||
# Stop and remove cancelled proxies.
|
|
||||||
@single.each do |k, v|
|
|
||||||
next if proxies.key?(k) && v.params == proxies[k]
|
|
||||||
|
|
||||||
@single.delete(k)&.stop
|
|
||||||
end
|
|
||||||
|
|
||||||
# Create and start missing proxies.
|
|
||||||
proxies.each do |k, v|
|
|
||||||
next if @single.key?(k)
|
|
||||||
|
|
||||||
(@single[k] = Single.new(*v)).run
|
|
||||||
rescue StandardError => e
|
|
||||||
$logger.error(self) do
|
|
||||||
e.message
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
Daemons.run_proc($config[:app_name], :dir => RUN_LOCATION, :log_dir => LOG_LOCATION) do
|
|
||||||
CustomLogger = Console::Filter[:debug => 0, :info => 1, :warn => 2, :error => 3]
|
|
||||||
logfile = File.open "#{LOG_LOCATION}/#{$config[:app_name]}.log", 'a'
|
|
||||||
logfile.sync = true
|
|
||||||
serialized = Console::Serialized::Logger.new logfile
|
|
||||||
$logger = CustomLogger.new serialized, :level => 0
|
|
||||||
|
|
||||||
# The "CONFIGURATION_FILE" is updated during the host sync procedure.
|
|
||||||
begin
|
|
||||||
$config.merge! YAML.load_file(CONFIGURATION_FILE)
|
|
||||||
rescue StandardError => e
|
|
||||||
$logger.error(self) do
|
|
||||||
"Error parsing config file #{CONFIGURATION_FILE}: #{e.message}"
|
|
||||||
end
|
|
||||||
exit(-1)
|
|
||||||
end
|
|
||||||
|
|
||||||
$logger = CustomLogger.new serialized, :level => LOG_LEVEL_MAP[$config[:tproxy_debug_level]]
|
|
||||||
|
|
||||||
VNMMAD::TProxy::Multi.new.run
|
|
||||||
end
|
|
||||||
@ -1,368 +0,0 @@
|
|||||||
# -------------------------------------------------------------------------- #
|
|
||||||
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
|
|
||||||
# #
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
|
||||||
# not use this file except in compliance with the License. You may obtain #
|
|
||||||
# a copy of the License at #
|
|
||||||
# #
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 #
|
|
||||||
# #
|
|
||||||
# Unless required by applicable law or agreed to in writing, software #
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, #
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
|
||||||
# See the License for the specific language governing permissions and #
|
|
||||||
# limitations under the License. #
|
|
||||||
#--------------------------------------------------------------------------- #
|
|
||||||
|
|
||||||
require 'base64'
|
|
||||||
require 'erb'
|
|
||||||
require 'json'
|
|
||||||
require 'open3'
|
|
||||||
require 'resolv'
|
|
||||||
|
|
||||||
module VNMMAD
|
|
||||||
|
|
||||||
# Module to handle transparent proxies.
|
|
||||||
module TProxy
|
|
||||||
|
|
||||||
# A helper class to manage the tproxy mapping in nftables.
|
|
||||||
class ProxyMap
|
|
||||||
|
|
||||||
def initialize(port_range = (16_000...32_000))
|
|
||||||
@range = port_range
|
|
||||||
reload
|
|
||||||
end
|
|
||||||
|
|
||||||
def reload
|
|
||||||
@map = self.class.load_proxy_map
|
|
||||||
@ports = self.class.load_proxy_ports(@map)
|
|
||||||
end
|
|
||||||
|
|
||||||
def get_port(bridge, service_addr, service_port)
|
|
||||||
if (v = @map[[bridge, service_addr, service_port]]).nil?
|
|
||||||
next_port
|
|
||||||
else
|
|
||||||
v[0]
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# Delete all map entries that do not match current config (for a specific bridge).
|
|
||||||
# This is a best-effort type of operation.
|
|
||||||
def cleanup(bridge, keep: @ports)
|
|
||||||
to_delete = @map.each_with_object({}) do |(k, v), acc|
|
|
||||||
acc[k] = v if k[0] == bridge && !keep.include?(v[0])
|
|
||||||
end
|
|
||||||
|
|
||||||
TProxy.nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding), :term => false)
|
|
||||||
<%- to_delete.each do |k, v| -%>
|
|
||||||
delete element ip one_tproxy proxies { "<%= k[0] %>" . <%= k[1..(-1)].join(' . ') %> }
|
|
||||||
<%- end -%>
|
|
||||||
NFT
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.load_proxy_map
|
|
||||||
TProxy.nft_json('list map ip one_tproxy proxies', :term => false)
|
|
||||||
.dig(0, 'nftables')
|
|
||||||
&.find {|item| !item['map'].nil? }
|
|
||||||
&.dig('map', 'elem')
|
|
||||||
.to_h {|item| item.map(&:values).map(&:flatten) }
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.load_proxy_ports(proxy_map = get_proxy_map)
|
|
||||||
proxy_map.map {|_, v| v[0] }
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
|
|
||||||
def next_port
|
|
||||||
# This is not the most efficient implementation, but we can safely assume
|
|
||||||
# it is likely the number of proxy ports openned should be low.
|
|
||||||
if (v = @range.find {|port| !@ports.include?(port) }).nil?
|
|
||||||
nil
|
|
||||||
else
|
|
||||||
@ports << v
|
|
||||||
v
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
# The entry point for the tproxy feature.
|
|
||||||
def self.setup_tproxy(nic, direction)
|
|
||||||
# Short-circuit if no tproxy config is provided.
|
|
||||||
return if CONF[:tproxy].to_a.empty?
|
|
||||||
|
|
||||||
begin
|
|
||||||
nonce = Integer(nic[:network_id]) % 16_000
|
|
||||||
rescue ArgumentError
|
|
||||||
return
|
|
||||||
end
|
|
||||||
|
|
||||||
proxy_map = ProxyMap.new
|
|
||||||
|
|
||||||
proxy_opts = CONF[:tproxy].to_a.each_with_object([]) do |conf, acc|
|
|
||||||
next if !conf[:network_id].nil? \
|
|
||||||
&& Integer(conf[:network_id]) != Integer(nic[:network_id])
|
|
||||||
|
|
||||||
next if !conf[:network].nil? \
|
|
||||||
&& conf[:network] != nic[:network]
|
|
||||||
|
|
||||||
next if conf[:remote_addr].nil? || conf[:remote_port].nil?
|
|
||||||
|
|
||||||
opts = {
|
|
||||||
# Multiple proxies in the same VNET can reuse the same routing tables and rules.
|
|
||||||
:rt_in => 16_000 + nonce,
|
|
||||||
:mark_in => 16_000 + nonce,
|
|
||||||
:rt_out => 32_000 + nonce,
|
|
||||||
:mark_out => 32_000 + nonce,
|
|
||||||
|
|
||||||
:service_addr => conf[:service_addr] || '169.254.16.9',
|
|
||||||
:service_port => Integer(conf[:service_port] || conf[:remote_port]),
|
|
||||||
|
|
||||||
:remote_addr => conf[:remote_addr],
|
|
||||||
:remote_port => Integer(conf[:remote_port])
|
|
||||||
}
|
|
||||||
|
|
||||||
next unless opts[:service_addr] =~ Resolv::IPv4::Regex
|
|
||||||
|
|
||||||
next unless opts[:remote_addr] =~ Resolv::IPv4::Regex
|
|
||||||
|
|
||||||
opts[:proxy_port] = proxy_map.get_port(nic[:bridge],
|
|
||||||
opts[:service_addr],
|
|
||||||
opts[:service_port])
|
|
||||||
|
|
||||||
acc << opts
|
|
||||||
rescue ArgumentError
|
|
||||||
next
|
|
||||||
end
|
|
||||||
|
|
||||||
# Short-circuit if no valid config is recognized.
|
|
||||||
return if proxy_opts.empty?
|
|
||||||
|
|
||||||
if direction == :up
|
|
||||||
proxy_map.cleanup(nic[:bridge],
|
|
||||||
:keep => proxy_opts.map {|opts| opts[:proxy_port] })
|
|
||||||
|
|
||||||
enable_tproxy(nic, proxy_opts)
|
|
||||||
|
|
||||||
run_tproxy('start')
|
|
||||||
run_tproxy('reload')
|
|
||||||
else
|
|
||||||
run_tproxy('stop')
|
|
||||||
|
|
||||||
disable_tproxy(nic, proxy_opts)
|
|
||||||
|
|
||||||
proxy_map.cleanup(nic[:bridge], :keep => []) # delete all
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.enable_tproxy(nic, proxy_opts)
|
|
||||||
br = nic[:bridge]
|
|
||||||
|
|
||||||
# Get the MAC of the bridge to later use it in NFT rules.
|
|
||||||
mac = ip_json(<<~LINK).dig(0, 0, 'address')
|
|
||||||
link show dev #{br}
|
|
||||||
LINK
|
|
||||||
|
|
||||||
return if mac.nil?
|
|
||||||
|
|
||||||
# Completely disable reverse path filtering for the bridge at hand. This is required
|
|
||||||
# for both proxy-arp to work and routing the response packets back to VM guests.
|
|
||||||
sysctl(<<~SYSCTL)
|
|
||||||
net.ipv4.conf.all.rp_filter=0
|
|
||||||
SYSCTL
|
|
||||||
sysctl(<<~SYSCTL)
|
|
||||||
net.ipv4.conf.#{br}.rp_filter=0
|
|
||||||
SYSCTL
|
|
||||||
|
|
||||||
proxy_opts.each do |opts|
|
|
||||||
# Enable proxy-arp.
|
|
||||||
ip_neighbour_add_proxy(<<~NEIGHBOUR)
|
|
||||||
#{opts[:service_addr]} dev #{br}
|
|
||||||
NEIGHBOUR
|
|
||||||
|
|
||||||
# This is required for proxy-arp to work.
|
|
||||||
ip_route_replace(<<~ROUTE)
|
|
||||||
#{opts[:service_addr]} dev lo
|
|
||||||
ROUTE
|
|
||||||
|
|
||||||
ip_rule_add(<<~RULE)
|
|
||||||
fwmark #{opts[:mark_in]} lookup #{opts[:rt_in]}
|
|
||||||
RULE
|
|
||||||
|
|
||||||
ip_route_replace(<<~ROUTE)
|
|
||||||
local default dev lo table #{opts[:rt_in]}
|
|
||||||
ROUTE
|
|
||||||
|
|
||||||
ip_rule_add(<<~RULE)
|
|
||||||
fwmark #{opts[:mark_out]} lookup #{opts[:rt_out]}
|
|
||||||
RULE
|
|
||||||
|
|
||||||
# Inject response packets (marked with :mark_out) back into the bridge.
|
|
||||||
ip_route_replace(<<~ROUTE)
|
|
||||||
default dev #{br} table #{opts[:rt_out]}
|
|
||||||
ROUTE
|
|
||||||
end
|
|
||||||
|
|
||||||
# This nftables config can be considered to be a failsafe. In case where proxy-arp
|
|
||||||
# does not work for any reason, users can still manually set permanent ARP mapping
|
|
||||||
# `arp -s 169.254.16.9 00:00:00:00:00:00` inside VM guests. Setting the mapping avoids
|
|
||||||
# ARP resolution, then nftables overrides / corrects the destination MAC address.
|
|
||||||
nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding))
|
|
||||||
table bridge one_tproxy {
|
|
||||||
chain #{br} {
|
|
||||||
type filter hook prerouting priority dstnat; policy accept;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
flush chain bridge one_tproxy #{br}
|
|
||||||
|
|
||||||
table bridge one_tproxy {
|
|
||||||
chain #{br} {
|
|
||||||
<%- proxy_opts.each do |opts| -%>
|
|
||||||
meta ibrname "#{br}" \\
|
|
||||||
ip daddr <%= opts[:service_addr] %> \\
|
|
||||||
tcp dport <%= opts[:service_port] %> \\
|
|
||||||
counter \\
|
|
||||||
meta pkttype set host ether daddr set #{mac} \\
|
|
||||||
accept
|
|
||||||
<%- end -%>
|
|
||||||
}
|
|
||||||
}
|
|
||||||
NFT
|
|
||||||
|
|
||||||
# The tproxy.rb process reads its config from the "ip one_tproxy proxies" map
|
|
||||||
# defined in nftables, that way users can manually restart the proxy on demand
|
|
||||||
# without the need of providing any command line arguments. The map is managed
|
|
||||||
# by the driver, proxy only reads its contents.
|
|
||||||
nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding))
|
|
||||||
table ip one_tproxy {
|
|
||||||
map proxies {
|
|
||||||
type ifname . ipv4_addr . inet_service : inet_service . ipv4_addr . inet_service . mark;
|
|
||||||
elements = {
|
|
||||||
<%- proxy_opts.each do |opts| -%>
|
|
||||||
"#{br}" . <%= opts[:service_addr] %> . <%= opts[:service_port] %> \\
|
|
||||||
: <%= opts[:proxy_port] %> . <%= opts[:remote_addr] %> . <%= opts[:remote_port] %> . <%= opts[:mark_out] %>,
|
|
||||||
<%- end -%>
|
|
||||||
}
|
|
||||||
}
|
|
||||||
chain #{br} {
|
|
||||||
type filter hook prerouting priority mangle; policy accept;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
flush chain ip one_tproxy #{br}
|
|
||||||
|
|
||||||
table ip one_tproxy {
|
|
||||||
chain #{br} {
|
|
||||||
<%- proxy_opts.each do |opts| -%>
|
|
||||||
iifname "#{br}" \\
|
|
||||||
meta l4proto tcp \\
|
|
||||||
ip daddr <%= opts[:service_addr] %> \\
|
|
||||||
tcp dport <%= opts[:service_port] %> \\
|
|
||||||
counter \\
|
|
||||||
mark set <%= opts[:mark_in] %> \\
|
|
||||||
tproxy to 127.0.0.1:<%= opts[:proxy_port] %>
|
|
||||||
<%- end -%>
|
|
||||||
}
|
|
||||||
}
|
|
||||||
NFT
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.disable_tproxy(nic, proxy_opts)
|
|
||||||
br = nic[:bridge]
|
|
||||||
|
|
||||||
proxy_opts.each do |opts|
|
|
||||||
ip_rule_del(<<~RULE)
|
|
||||||
fwmark #{opts[:mark_in]} lookup #{opts[:rt_in]}
|
|
||||||
RULE
|
|
||||||
|
|
||||||
ip_rule_del(<<~RULE)
|
|
||||||
fwmark #{opts[:mark_out]} lookup #{opts[:rt_out]}
|
|
||||||
RULE
|
|
||||||
end
|
|
||||||
|
|
||||||
nft(<<~NFT)
|
|
||||||
table bridge one_tproxy {
|
|
||||||
chain #{br} {
|
|
||||||
type filter hook prerouting priority dstnat; policy accept;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
flush chain bridge one_tproxy #{br}
|
|
||||||
NFT
|
|
||||||
|
|
||||||
nft(<<~NFT)
|
|
||||||
table ip one_tproxy {
|
|
||||||
chain #{br} {
|
|
||||||
type filter hook prerouting priority mangle; policy accept;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
flush chain ip one_tproxy #{br}
|
|
||||||
NFT
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.run_tproxy(cmd)
|
|
||||||
run(:tproxy, *cmd.strip.split(' '))
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.ip_json(cmd, **opts)
|
|
||||||
o, e, s = run(:ip_unpriv, '-j', *cmd.strip.split(' '), **opts)
|
|
||||||
if s.success?
|
|
||||||
[JSON.parse(o), e, s]
|
|
||||||
else
|
|
||||||
[{}, e, s]
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.ip_neighbour_add_proxy(cmd)
|
|
||||||
run(:ip, 'neighbour', 'add', 'proxy', *cmd.strip.split(' '))
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.ip_rule_add(cmd)
|
|
||||||
args = cmd.strip.split(' ')
|
|
||||||
if (check = run(:ip, 'rule', 'list', *args))[0].strip.empty?
|
|
||||||
run(:ip, 'rule', 'add', *args)
|
|
||||||
else
|
|
||||||
check
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.ip_rule_del(cmd)
|
|
||||||
args = cmd.strip.split(' ')
|
|
||||||
if (check = run(:ip, 'rule', 'list', *args))[0].strip.empty?
|
|
||||||
check
|
|
||||||
else
|
|
||||||
run(:ip, 'rule', 'del', *args)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.ip_route_replace(cmd)
|
|
||||||
run(:ip, 'route', 'replace', *cmd.strip.split(' '))
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.nft_json(cmd, **opts)
|
|
||||||
o, e, s = run(:nft, '-j', *cmd.strip.split(' '), **opts)
|
|
||||||
if s.success?
|
|
||||||
[JSON.parse(o), e, s]
|
|
||||||
else
|
|
||||||
[{}, e, s]
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.nft(script, **opts)
|
|
||||||
run(:nft, '-f-', **opts, :stdin_data => script)
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.sysctl(cmd)
|
|
||||||
run(:sysctl, *cmd.strip.split(' '))
|
|
||||||
end
|
|
||||||
|
|
||||||
private_class_method def self.run(sym, *args, **opts)
|
|
||||||
VNMNetwork::Command.no_shell(sym, *args, **opts)
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
@ -53,9 +53,6 @@ module VNMMAD
|
|||||||
# Create the bridge.
|
# Create the bridge.
|
||||||
create_bridge(@nic)
|
create_bridge(@nic)
|
||||||
|
|
||||||
# Setup transparent proxies.
|
|
||||||
TProxy.setup_tproxy(@nic, :up)
|
|
||||||
|
|
||||||
# Check that no other vlans are connected to this bridge
|
# Check that no other vlans are connected to this bridge
|
||||||
validate_vlan_id if @nic[:conf][:validate_vlan_id]
|
validate_vlan_id if @nic[:conf][:validate_vlan_id]
|
||||||
|
|
||||||
@ -134,9 +131,6 @@ module VNMMAD
|
|||||||
|
|
||||||
@bridges[@nic[:bridge]].delete(@nic[:vlan_dev])
|
@bridges[@nic[:bridge]].delete(@nic[:vlan_dev])
|
||||||
|
|
||||||
# Setup transparent proxies.
|
|
||||||
TProxy.setup_tproxy(@nic, :down)
|
|
||||||
|
|
||||||
# Delete the bridge.
|
# Delete the bridge.
|
||||||
OpenNebula.exec_and_log("#{command(:ip)} link delete"\
|
OpenNebula.exec_and_log("#{command(:ip)} link delete"\
|
||||||
" #{@nic[:bridge]}")
|
" #{@nic[:bridge]}")
|
||||||
|
|||||||
@ -33,7 +33,6 @@ require 'sg_driver'
|
|||||||
require 'vlan'
|
require 'vlan'
|
||||||
require 'no_vlan'
|
require 'no_vlan'
|
||||||
require 'scripts_common'
|
require 'scripts_common'
|
||||||
require 'tproxy'
|
|
||||||
|
|
||||||
Dir[File.expand_path('vnmmad-load.d', File.dirname(__FILE__)) + "/*.rb"].each{ |f| require f }
|
Dir[File.expand_path('vnmmad-load.d', File.dirname(__FILE__)) + "/*.rb"].each{ |f| require f }
|
||||||
|
|
||||||
@ -53,10 +52,7 @@ rescue
|
|||||||
:vlan_mtu => '1500',
|
:vlan_mtu => '1500',
|
||||||
:ipset_maxelem => '65536',
|
:ipset_maxelem => '65536',
|
||||||
:keep_empty_bridge => false,
|
:keep_empty_bridge => false,
|
||||||
:datastore_location => '/var/lib/one/datastores',
|
:datastore_location => '/var/lib/one/datastores'
|
||||||
:tproxy_debug_level => 2, # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
|
|
||||||
:tproxy_process_owner => 'oneadmin',
|
|
||||||
:tproxy => []
|
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user