mirror of
https://github.com/OpenNebula/one.git
synced 2024-12-22 13:33:52 +03:00
M #-: Remove tproxy logic
This commit is contained in:
Ruben S. Montero
parent
b7d070459b
commit
f194c1241f
@ -1538,9 +1538,7 @@ NETWORK_FILES="src/vnm_mad/remotes/lib/vnm_driver.rb \
|
||||
src/vnm_mad/remotes/lib/no_vlan.rb \
|
||||
src/vnm_mad/remotes/lib/security_groups.rb \
|
||||
src/vnm_mad/remotes/lib/security_groups_iptables.rb \
|
||||
src/vnm_mad/remotes/lib/nic.rb \
|
||||
src/vnm_mad/remotes/lib/tproxy \
|
||||
src/vnm_mad/remotes/lib/tproxy.rb"
|
||||
src/vnm_mad/remotes/lib/nic.rb"
|
||||
|
||||
NETWORK_8021Q_FILES="src/vnm_mad/remotes/802.1Q/clean \
|
||||
src/vnm_mad/remotes/802.1Q/post \
|
||||
|
||||
@ -6,7 +6,7 @@ Cmnd_Alias ONE_HA = /usr/bin/systemctl start opennebula-flow, /usr/bin/systemctl
|
||||
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
|
||||
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
|
||||
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
|
||||
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *, /usr/sbin/nft, /usr/sbin/sysctl net.ipv4.conf.*, /var/tmp/one/vnm/tproxy
|
||||
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *
|
||||
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
|
||||
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu
|
||||
|
||||
@ -6,7 +6,7 @@ Cmnd_Alias ONE_HA = /usr/bin/systemctl start opennebula-flow, /usr/bin/systemctl
|
||||
Cmnd_Alias ONE_LVM = /usr/sbin/lvcreate, /usr/sbin/lvremove, /usr/sbin/lvs, /usr/sbin/vgdisplay, /usr/sbin/lvchange, /usr/sbin/lvscan, /usr/sbin/lvextend
|
||||
Cmnd_Alias ONE_LXC = /usr/bin/mount, /usr/bin/umount, /usr/bin/bindfs, /usr/sbin/losetup, /usr/bin/qemu-nbd, /usr/bin/lxc-attach, /usr/bin/lxc-config, /usr/bin/lxc-create, /usr/bin/lxc-destroy, /usr/bin/lxc-info, /usr/bin/lxc-ls, /usr/bin/lxc-start, /usr/bin/lxc-stop, /usr/bin/lxc-console, /usr/sbin/e2fsck, /usr/sbin/resize2fs, /usr/sbin/xfs_growfs, /usr/bin/rbd-nbd
|
||||
Cmnd_Alias ONE_MARKET = /usr/lib/one/sh/create_container_image.sh
|
||||
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *, /usr/sbin/nft, /usr/sbin/sysctl net.ipv4.conf.*, /var/tmp/one/vnm/tproxy
|
||||
Cmnd_Alias ONE_NET = /usr/sbin/ebtables, /usr/sbin/iptables, /usr/sbin/ip6tables, /usr/sbin/ipset, /usr/sbin/ip link *, /usr/sbin/ip neighbour *, /usr/sbin/ip route *, /usr/sbin/ip rule *, /usr/sbin/ip tuntap *
|
||||
Cmnd_Alias ONE_OVS = /usr/bin/ovs-ofctl, /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl
|
||||
Cmnd_Alias ONE_MEM = /usr/sbin/sysctl vm.drop_caches=3 vm.compact_memory=1
|
||||
Cmnd_Alias ONE_VGPU = /var/tmp/one/vgpu
|
||||
|
||||
@ -33,10 +33,7 @@ class Sudoers
|
||||
'ip neighbour *',
|
||||
'ip route *',
|
||||
'ip rule *',
|
||||
'ip tuntap *',
|
||||
'nft',
|
||||
'sysctl net.ipv4.conf.*',
|
||||
'/var/tmp/one/vnm/tproxy'
|
||||
'ip tuntap *'
|
||||
],
|
||||
:LVM => [
|
||||
'lvcreate', 'lvremove', 'lvs', 'vgdisplay', 'lvchange', 'lvscan', 'lvextend'
|
||||
|
||||
@ -97,52 +97,3 @@
|
||||
# :ip_link_conf:
|
||||
# :udp6zerocsumrx:
|
||||
# :tos: 3
|
||||
|
||||
################################################################################
|
||||
# TProxy / OneGate Options
|
||||
################################################################################
|
||||
|
||||
# Each entry in the :tproxy array is a definition of a proxy instance:
|
||||
#
|
||||
# Link-local address and port used by VM guests to access the proxy.
|
||||
# :service_addr: 169.254.16.9
|
||||
# :service_port: 5030
|
||||
#
|
||||
# IP address and port of a service to proxy connections to.
|
||||
# It must be accessible from the hypervisor hosts.
|
||||
# :remote_addr: 10.11.12.13
|
||||
# :remote_port: 5030
|
||||
#
|
||||
# Optionally a VNET can be specified by name or id.
|
||||
# If it's not defined (the default behavior), then the proxy code will be
|
||||
# applied to all VNETs / NICs of all involved VM guests.
|
||||
# :network: vnet
|
||||
# :network_id: 0
|
||||
#
|
||||
# The simplest example of an OneGate proxy config applied to all VNETs:
|
||||
# :tproxy:
|
||||
# - :service_addr: 169.254.16.9
|
||||
# :service_port: 5030
|
||||
# :remote_addr: 10.11.12.13 # HA VIP
|
||||
# :remote_port: 5030
|
||||
|
||||
# Default log level for the proxy process.
|
||||
# Logs are located on hypervisor hosts at /var/log/one_tproxy.log.
|
||||
# The proxy process can be restarted on demand (as root): `/var/tmp/one/vnm/tproxy restart`.
|
||||
#:tproxy_debug_level: 2 # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
|
||||
|
||||
#:tproxy:
|
||||
#- :service_addr: 169.254.16.9
|
||||
# :service_port: 5030
|
||||
# :remote_addr: 10.11.12.13
|
||||
# :remote_port: 5030
|
||||
#- :network: vnet0
|
||||
# :service_addr: 169.254.16.9
|
||||
# :service_port: 4321
|
||||
# :remote_addr: 10.11.12.13
|
||||
# :remote_port: 1234
|
||||
#- :network_id: 0
|
||||
# :service_addr: 169.254.16.9
|
||||
# :service_port: 4321
|
||||
# :remote_addr: 10.11.12.13
|
||||
# :remote_port: 1234
|
||||
|
||||
@ -35,10 +35,7 @@ module VNMMAD
|
||||
:ovs_ofctl => 'sudo -n ovs-ofctl',
|
||||
:ovs_appctl => 'sudo -n ovs-appctl',
|
||||
:lsmod => 'lsmod',
|
||||
:ipset => 'sudo -n ipset',
|
||||
:nft => 'sudo -n nft',
|
||||
:sysctl => 'sudo -n sysctl',
|
||||
:tproxy => 'sudo -n /var/tmp/one/vnm/tproxy'
|
||||
:ipset => 'sudo -n ipset'
|
||||
}
|
||||
|
||||
# Adjust :ip[6]tables commands to work with legacy versions
|
||||
|
||||
@ -42,9 +42,6 @@ module VNMMAD
|
||||
# Create the bridge.
|
||||
create_bridge(@nic)
|
||||
|
||||
# Setup transparent proxies.
|
||||
TProxy.setup_tproxy(@nic, :up)
|
||||
|
||||
# Skip if vlan device is already in the bridge.
|
||||
next if !@nic[:phydev] || @nic[:phydev].empty? ||
|
||||
@bridges[@nic[:bridge]].include?(@nic[:phydev])
|
||||
@ -99,9 +96,6 @@ module VNMMAD
|
||||
|
||||
next if keep
|
||||
|
||||
# Setup transparent proxies.
|
||||
TProxy.setup_tproxy(@nic, :down)
|
||||
|
||||
# Delete the bridge.
|
||||
OpenNebula.exec_and_log("#{command(:ip)} link delete #{@nic[:bridge]}")
|
||||
|
||||
|
||||
@ -1,272 +0,0 @@
|
||||
#!/usr/bin/env ruby
|
||||
# -------------------------------------------------------------------------- #
|
||||
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
||||
# not use this file except in compliance with the License. You may obtain #
|
||||
# a copy of the License at #
|
||||
# #
|
||||
# http://www.apache.org/licenses/LICENSE-2.0 #
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software #
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, #
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
||||
# See the License for the specific language governing permissions and #
|
||||
# limitations under the License. #
|
||||
#--------------------------------------------------------------------------- #
|
||||
|
||||
# frozen_string_literal: true
|
||||
|
||||
# rubocop:disable Lint/MissingCopEnableDirective
|
||||
# rubocop:disable Metrics/ParameterLists
|
||||
# rubocop:disable Style/Documentation
|
||||
# rubocop:disable Style/GlobalVars
|
||||
# rubocop:disable Style/ParallelAssignment
|
||||
|
||||
RUBY_LIB_LOCATION = '/usr/lib/one/ruby'
|
||||
GEMS_LOCATION = '/usr/share/one/gems'
|
||||
LOG_LOCATION = '/var/log'
|
||||
RUN_LOCATION = '/var/run'
|
||||
REMOTES_LOCATION = '/var/tmp/one'
|
||||
CONFIGURATION_FILE = REMOTES_LOCATION + '/etc/vnm/OpenNebulaNetwork.conf'
|
||||
|
||||
# %%RUBYGEMS_SETUP_BEGIN%%
|
||||
if File.directory?(GEMS_LOCATION)
|
||||
real_gems_path = File.realpath(GEMS_LOCATION)
|
||||
if !defined?(Gem) || Gem.path != [real_gems_path]
|
||||
$LOAD_PATH.reject! {|p| p =~ /vendor_ruby/ }
|
||||
|
||||
# Suppress warnings from Rubygems
|
||||
# https://github.com/OpenNebula/one/issues/5379
|
||||
begin
|
||||
verb = $VERBOSE
|
||||
$VERBOSE = nil
|
||||
require 'rubygems'
|
||||
Gem.use_paths(real_gems_path)
|
||||
ensure
|
||||
$VERBOSE = verb
|
||||
end
|
||||
end
|
||||
end
|
||||
# %%RUBYGEMS_SETUP_END%%
|
||||
|
||||
$LOAD_PATH << RUBY_LIB_LOCATION
|
||||
|
||||
require 'async/io'
|
||||
require 'async/io/stream'
|
||||
require 'async/io/trap'
|
||||
require 'console'
|
||||
require 'daemons'
|
||||
require 'json'
|
||||
require 'open3'
|
||||
require 'socket'
|
||||
require 'yaml'
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
:tproxy_debug_level => 2, # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
|
||||
:app_name => 'one_tproxy'
|
||||
}.freeze
|
||||
|
||||
LOG_LEVEL_MAP = {
|
||||
0 => 3, # ERROR
|
||||
1 => 2, # WARN
|
||||
2 => 1, # INFO
|
||||
3 => 0 # DEBUG
|
||||
}.freeze
|
||||
|
||||
$config = DEFAULT_CONFIG.dup
|
||||
$logger = nil
|
||||
|
||||
module VNMMAD
|
||||
|
||||
module TProxy
|
||||
|
||||
# A single async TCP transparent proxy implementation, it binds to a single port and
|
||||
# marks outgoing packets with SO_MARK.
|
||||
class Single
|
||||
|
||||
def initialize(bport, daddr, dport, smark)
|
||||
@bport, @daddr, @dport, @smark = bport, daddr, dport, smark
|
||||
|
||||
@socket = setup_socket('127.0.0.1', @bport, @smark)
|
||||
@proxy_ep = Async::IO::Endpoint.socket(@socket)
|
||||
end
|
||||
|
||||
def params
|
||||
[@bport, @daddr, @dport, @smark]
|
||||
end
|
||||
|
||||
def run
|
||||
Async do |task|
|
||||
glue_peers(@task = task)
|
||||
end
|
||||
end
|
||||
|
||||
def stop
|
||||
$logger.info(self) do
|
||||
"Stop #{Addrinfo.tcp('127.0.0.1', @bport).inspect}"
|
||||
end
|
||||
@socket.close
|
||||
@task.stop
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def setup_socket(baddr, bport, smark, listen = Socket::SOMAXCONN)
|
||||
sock = Socket.new Socket::AF_INET, Socket::SOCK_STREAM, 0
|
||||
|
||||
sock.setsockopt Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1
|
||||
sock.setsockopt Socket::SOL_SOCKET, Socket::SO_MARK, smark
|
||||
|
||||
sock.setsockopt Socket::SOL_IP, Socket::IP_TRANSPARENT, 1
|
||||
|
||||
$logger.info(self) do
|
||||
"Bind #{Addrinfo.tcp(baddr, bport).inspect}"
|
||||
end
|
||||
|
||||
sock.bind Socket.pack_sockaddr_in(bport, baddr)
|
||||
sock.listen listen
|
||||
sock
|
||||
end
|
||||
|
||||
def glue_streams(stream1, stream2, task)
|
||||
task.async do |subtask|
|
||||
concurrent = []
|
||||
concurrent << subtask.async do
|
||||
while (chunk = stream1.read_partial)
|
||||
stream2.write chunk
|
||||
stream2.flush
|
||||
end
|
||||
end
|
||||
concurrent << subtask.async do
|
||||
while (chunk = stream2.read_partial)
|
||||
stream1.write chunk
|
||||
stream1.flush
|
||||
end
|
||||
end
|
||||
concurrent.each(&:wait)
|
||||
end
|
||||
end
|
||||
|
||||
def glue_peers(task)
|
||||
@proxy_ep.accept do |client_peer|
|
||||
$logger.debug(self) do
|
||||
"Accept #{client_peer.remote_address.inspect}"
|
||||
end
|
||||
|
||||
begin
|
||||
server_ep = Async::IO::Endpoint.tcp @daddr,
|
||||
@dport
|
||||
server_ep.connect do |server_peer|
|
||||
client_stream, server_stream = Async::IO::Stream.new(client_peer),
|
||||
Async::IO::Stream.new(server_peer)
|
||||
|
||||
glue_streams(client_stream, server_stream, task).wait
|
||||
ensure
|
||||
$logger.debug(self) do
|
||||
"Close #{server_peer.remote_address.inspect}"
|
||||
end
|
||||
|
||||
server_peer.close
|
||||
end
|
||||
rescue Errno::ECONNREFUSED,
|
||||
Errno::ECONNRESET,
|
||||
Errno::EHOSTUNREACH,
|
||||
Errno::ETIMEDOUT => e
|
||||
$logger.error(self) do
|
||||
e.message
|
||||
end
|
||||
end
|
||||
ensure
|
||||
$logger.debug(self) do
|
||||
"Close #{client_peer.remote_address.inspect}"
|
||||
end
|
||||
|
||||
client_peer.close
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
# Combine multiple proxies into one async service.
|
||||
class Multi
|
||||
|
||||
def initialize
|
||||
@single = {}
|
||||
@sighup = Async::IO::Trap.new :HUP
|
||||
@sighup.install!
|
||||
end
|
||||
|
||||
def run
|
||||
Async do
|
||||
reload
|
||||
@sighup.wait { reload }
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def reload
|
||||
cmd = "nft -j list map ip #{$config[:app_name]} proxies"
|
||||
|
||||
o, e, s = Open3.capture3(*cmd.split(' '))
|
||||
|
||||
unless s.success?
|
||||
$logger.error(self) do
|
||||
"Error getting proxy map from nftables: #{e}"
|
||||
end
|
||||
return
|
||||
end
|
||||
|
||||
proxies = JSON.parse(o)
|
||||
&.dig('nftables')
|
||||
&.find {|item| !item['map'].nil? }
|
||||
&.dig('map', 'elem')
|
||||
.to_h {|item| item.map(&:values).map(&:flatten) }
|
||||
|
||||
# Stop and remove cancelled proxies.
|
||||
@single.each do |k, v|
|
||||
next if proxies.key?(k) && v.params == proxies[k]
|
||||
|
||||
@single.delete(k)&.stop
|
||||
end
|
||||
|
||||
# Create and start missing proxies.
|
||||
proxies.each do |k, v|
|
||||
next if @single.key?(k)
|
||||
|
||||
(@single[k] = Single.new(*v)).run
|
||||
rescue StandardError => e
|
||||
$logger.error(self) do
|
||||
e.message
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
Daemons.run_proc($config[:app_name], :dir => RUN_LOCATION, :log_dir => LOG_LOCATION) do
|
||||
CustomLogger = Console::Filter[:debug => 0, :info => 1, :warn => 2, :error => 3]
|
||||
logfile = File.open "#{LOG_LOCATION}/#{$config[:app_name]}.log", 'a'
|
||||
logfile.sync = true
|
||||
serialized = Console::Serialized::Logger.new logfile
|
||||
$logger = CustomLogger.new serialized, :level => 0
|
||||
|
||||
# The "CONFIGURATION_FILE" is updated during the host sync procedure.
|
||||
begin
|
||||
$config.merge! YAML.load_file(CONFIGURATION_FILE)
|
||||
rescue StandardError => e
|
||||
$logger.error(self) do
|
||||
"Error parsing config file #{CONFIGURATION_FILE}: #{e.message}"
|
||||
end
|
||||
exit(-1)
|
||||
end
|
||||
|
||||
$logger = CustomLogger.new serialized, :level => LOG_LEVEL_MAP[$config[:tproxy_debug_level]]
|
||||
|
||||
VNMMAD::TProxy::Multi.new.run
|
||||
end
|
||||
@ -1,368 +0,0 @@
|
||||
# -------------------------------------------------------------------------- #
|
||||
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
||||
# not use this file except in compliance with the License. You may obtain #
|
||||
# a copy of the License at #
|
||||
# #
|
||||
# http://www.apache.org/licenses/LICENSE-2.0 #
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software #
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, #
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
||||
# See the License for the specific language governing permissions and #
|
||||
# limitations under the License. #
|
||||
#--------------------------------------------------------------------------- #
|
||||
|
||||
require 'base64'
|
||||
require 'erb'
|
||||
require 'json'
|
||||
require 'open3'
|
||||
require 'resolv'
|
||||
|
||||
module VNMMAD
|
||||
|
||||
# Module to handle transparent proxies.
|
||||
module TProxy
|
||||
|
||||
# A helper class to manage the tproxy mapping in nftables.
|
||||
class ProxyMap
|
||||
|
||||
def initialize(port_range = (16_000...32_000))
|
||||
@range = port_range
|
||||
reload
|
||||
end
|
||||
|
||||
def reload
|
||||
@map = self.class.load_proxy_map
|
||||
@ports = self.class.load_proxy_ports(@map)
|
||||
end
|
||||
|
||||
def get_port(bridge, service_addr, service_port)
|
||||
if (v = @map[[bridge, service_addr, service_port]]).nil?
|
||||
next_port
|
||||
else
|
||||
v[0]
|
||||
end
|
||||
end
|
||||
|
||||
# Delete all map entries that do not match current config (for a specific bridge).
|
||||
# This is a best-effort type of operation.
|
||||
def cleanup(bridge, keep: @ports)
|
||||
to_delete = @map.each_with_object({}) do |(k, v), acc|
|
||||
acc[k] = v if k[0] == bridge && !keep.include?(v[0])
|
||||
end
|
||||
|
||||
TProxy.nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding), :term => false)
|
||||
<%- to_delete.each do |k, v| -%>
|
||||
delete element ip one_tproxy proxies { "<%= k[0] %>" . <%= k[1..(-1)].join(' . ') %> }
|
||||
<%- end -%>
|
||||
NFT
|
||||
end
|
||||
|
||||
def self.load_proxy_map
|
||||
TProxy.nft_json('list map ip one_tproxy proxies', :term => false)
|
||||
.dig(0, 'nftables')
|
||||
&.find {|item| !item['map'].nil? }
|
||||
&.dig('map', 'elem')
|
||||
.to_h {|item| item.map(&:values).map(&:flatten) }
|
||||
end
|
||||
|
||||
def self.load_proxy_ports(proxy_map = get_proxy_map)
|
||||
proxy_map.map {|_, v| v[0] }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def next_port
|
||||
# This is not the most efficient implementation, but we can safely assume
|
||||
# it is likely the number of proxy ports openned should be low.
|
||||
if (v = @range.find {|port| !@ports.include?(port) }).nil?
|
||||
nil
|
||||
else
|
||||
@ports << v
|
||||
v
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
# The entry point for the tproxy feature.
|
||||
def self.setup_tproxy(nic, direction)
|
||||
# Short-circuit if no tproxy config is provided.
|
||||
return if CONF[:tproxy].to_a.empty?
|
||||
|
||||
begin
|
||||
nonce = Integer(nic[:network_id]) % 16_000
|
||||
rescue ArgumentError
|
||||
return
|
||||
end
|
||||
|
||||
proxy_map = ProxyMap.new
|
||||
|
||||
proxy_opts = CONF[:tproxy].to_a.each_with_object([]) do |conf, acc|
|
||||
next if !conf[:network_id].nil? \
|
||||
&& Integer(conf[:network_id]) != Integer(nic[:network_id])
|
||||
|
||||
next if !conf[:network].nil? \
|
||||
&& conf[:network] != nic[:network]
|
||||
|
||||
next if conf[:remote_addr].nil? || conf[:remote_port].nil?
|
||||
|
||||
opts = {
|
||||
# Multiple proxies in the same VNET can reuse the same routing tables and rules.
|
||||
:rt_in => 16_000 + nonce,
|
||||
:mark_in => 16_000 + nonce,
|
||||
:rt_out => 32_000 + nonce,
|
||||
:mark_out => 32_000 + nonce,
|
||||
|
||||
:service_addr => conf[:service_addr] || '169.254.16.9',
|
||||
:service_port => Integer(conf[:service_port] || conf[:remote_port]),
|
||||
|
||||
:remote_addr => conf[:remote_addr],
|
||||
:remote_port => Integer(conf[:remote_port])
|
||||
}
|
||||
|
||||
next unless opts[:service_addr] =~ Resolv::IPv4::Regex
|
||||
|
||||
next unless opts[:remote_addr] =~ Resolv::IPv4::Regex
|
||||
|
||||
opts[:proxy_port] = proxy_map.get_port(nic[:bridge],
|
||||
opts[:service_addr],
|
||||
opts[:service_port])
|
||||
|
||||
acc << opts
|
||||
rescue ArgumentError
|
||||
next
|
||||
end
|
||||
|
||||
# Short-circuit if no valid config is recognized.
|
||||
return if proxy_opts.empty?
|
||||
|
||||
if direction == :up
|
||||
proxy_map.cleanup(nic[:bridge],
|
||||
:keep => proxy_opts.map {|opts| opts[:proxy_port] })
|
||||
|
||||
enable_tproxy(nic, proxy_opts)
|
||||
|
||||
run_tproxy('start')
|
||||
run_tproxy('reload')
|
||||
else
|
||||
run_tproxy('stop')
|
||||
|
||||
disable_tproxy(nic, proxy_opts)
|
||||
|
||||
proxy_map.cleanup(nic[:bridge], :keep => []) # delete all
|
||||
end
|
||||
end
|
||||
|
||||
def self.enable_tproxy(nic, proxy_opts)
|
||||
br = nic[:bridge]
|
||||
|
||||
# Get the MAC of the bridge to later use it in NFT rules.
|
||||
mac = ip_json(<<~LINK).dig(0, 0, 'address')
|
||||
link show dev #{br}
|
||||
LINK
|
||||
|
||||
return if mac.nil?
|
||||
|
||||
# Completely disable reverse path filtering for the bridge at hand. This is required
|
||||
# for both proxy-arp to work and routing the response packets back to VM guests.
|
||||
sysctl(<<~SYSCTL)
|
||||
net.ipv4.conf.all.rp_filter=0
|
||||
SYSCTL
|
||||
sysctl(<<~SYSCTL)
|
||||
net.ipv4.conf.#{br}.rp_filter=0
|
||||
SYSCTL
|
||||
|
||||
proxy_opts.each do |opts|
|
||||
# Enable proxy-arp.
|
||||
ip_neighbour_add_proxy(<<~NEIGHBOUR)
|
||||
#{opts[:service_addr]} dev #{br}
|
||||
NEIGHBOUR
|
||||
|
||||
# This is required for proxy-arp to work.
|
||||
ip_route_replace(<<~ROUTE)
|
||||
#{opts[:service_addr]} dev lo
|
||||
ROUTE
|
||||
|
||||
ip_rule_add(<<~RULE)
|
||||
fwmark #{opts[:mark_in]} lookup #{opts[:rt_in]}
|
||||
RULE
|
||||
|
||||
ip_route_replace(<<~ROUTE)
|
||||
local default dev lo table #{opts[:rt_in]}
|
||||
ROUTE
|
||||
|
||||
ip_rule_add(<<~RULE)
|
||||
fwmark #{opts[:mark_out]} lookup #{opts[:rt_out]}
|
||||
RULE
|
||||
|
||||
# Inject response packets (marked with :mark_out) back into the bridge.
|
||||
ip_route_replace(<<~ROUTE)
|
||||
default dev #{br} table #{opts[:rt_out]}
|
||||
ROUTE
|
||||
end
|
||||
|
||||
# This nftables config can be considered to be a failsafe. In case where proxy-arp
|
||||
# does not work for any reason, users can still manually set permanent ARP mapping
|
||||
# `arp -s 169.254.16.9 00:00:00:00:00:00` inside VM guests. Setting the mapping avoids
|
||||
# ARP resolution, then nftables overrides / corrects the destination MAC address.
|
||||
nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding))
|
||||
table bridge one_tproxy {
|
||||
chain #{br} {
|
||||
type filter hook prerouting priority dstnat; policy accept;
|
||||
}
|
||||
}
|
||||
|
||||
flush chain bridge one_tproxy #{br}
|
||||
|
||||
table bridge one_tproxy {
|
||||
chain #{br} {
|
||||
<%- proxy_opts.each do |opts| -%>
|
||||
meta ibrname "#{br}" \\
|
||||
ip daddr <%= opts[:service_addr] %> \\
|
||||
tcp dport <%= opts[:service_port] %> \\
|
||||
counter \\
|
||||
meta pkttype set host ether daddr set #{mac} \\
|
||||
accept
|
||||
<%- end -%>
|
||||
}
|
||||
}
|
||||
NFT
|
||||
|
||||
# The tproxy.rb process reads its config from the "ip one_tproxy proxies" map
|
||||
# defined in nftables, that way users can manually restart the proxy on demand
|
||||
# without the need of providing any command line arguments. The map is managed
|
||||
# by the driver, proxy only reads its contents.
|
||||
nft(ERB.new(<<~NFT, :trim_mode => '-').result(binding))
|
||||
table ip one_tproxy {
|
||||
map proxies {
|
||||
type ifname . ipv4_addr . inet_service : inet_service . ipv4_addr . inet_service . mark;
|
||||
elements = {
|
||||
<%- proxy_opts.each do |opts| -%>
|
||||
"#{br}" . <%= opts[:service_addr] %> . <%= opts[:service_port] %> \\
|
||||
: <%= opts[:proxy_port] %> . <%= opts[:remote_addr] %> . <%= opts[:remote_port] %> . <%= opts[:mark_out] %>,
|
||||
<%- end -%>
|
||||
}
|
||||
}
|
||||
chain #{br} {
|
||||
type filter hook prerouting priority mangle; policy accept;
|
||||
}
|
||||
}
|
||||
|
||||
flush chain ip one_tproxy #{br}
|
||||
|
||||
table ip one_tproxy {
|
||||
chain #{br} {
|
||||
<%- proxy_opts.each do |opts| -%>
|
||||
iifname "#{br}" \\
|
||||
meta l4proto tcp \\
|
||||
ip daddr <%= opts[:service_addr] %> \\
|
||||
tcp dport <%= opts[:service_port] %> \\
|
||||
counter \\
|
||||
mark set <%= opts[:mark_in] %> \\
|
||||
tproxy to 127.0.0.1:<%= opts[:proxy_port] %>
|
||||
<%- end -%>
|
||||
}
|
||||
}
|
||||
NFT
|
||||
end
|
||||
|
||||
def self.disable_tproxy(nic, proxy_opts)
|
||||
br = nic[:bridge]
|
||||
|
||||
proxy_opts.each do |opts|
|
||||
ip_rule_del(<<~RULE)
|
||||
fwmark #{opts[:mark_in]} lookup #{opts[:rt_in]}
|
||||
RULE
|
||||
|
||||
ip_rule_del(<<~RULE)
|
||||
fwmark #{opts[:mark_out]} lookup #{opts[:rt_out]}
|
||||
RULE
|
||||
end
|
||||
|
||||
nft(<<~NFT)
|
||||
table bridge one_tproxy {
|
||||
chain #{br} {
|
||||
type filter hook prerouting priority dstnat; policy accept;
|
||||
}
|
||||
}
|
||||
|
||||
flush chain bridge one_tproxy #{br}
|
||||
NFT
|
||||
|
||||
nft(<<~NFT)
|
||||
table ip one_tproxy {
|
||||
chain #{br} {
|
||||
type filter hook prerouting priority mangle; policy accept;
|
||||
}
|
||||
}
|
||||
|
||||
flush chain ip one_tproxy #{br}
|
||||
NFT
|
||||
end
|
||||
|
||||
def self.run_tproxy(cmd)
|
||||
run(:tproxy, *cmd.strip.split(' '))
|
||||
end
|
||||
|
||||
def self.ip_json(cmd, **opts)
|
||||
o, e, s = run(:ip_unpriv, '-j', *cmd.strip.split(' '), **opts)
|
||||
if s.success?
|
||||
[JSON.parse(o), e, s]
|
||||
else
|
||||
[{}, e, s]
|
||||
end
|
||||
end
|
||||
|
||||
def self.ip_neighbour_add_proxy(cmd)
|
||||
run(:ip, 'neighbour', 'add', 'proxy', *cmd.strip.split(' '))
|
||||
end
|
||||
|
||||
def self.ip_rule_add(cmd)
|
||||
args = cmd.strip.split(' ')
|
||||
if (check = run(:ip, 'rule', 'list', *args))[0].strip.empty?
|
||||
run(:ip, 'rule', 'add', *args)
|
||||
else
|
||||
check
|
||||
end
|
||||
end
|
||||
|
||||
def self.ip_rule_del(cmd)
|
||||
args = cmd.strip.split(' ')
|
||||
if (check = run(:ip, 'rule', 'list', *args))[0].strip.empty?
|
||||
check
|
||||
else
|
||||
run(:ip, 'rule', 'del', *args)
|
||||
end
|
||||
end
|
||||
|
||||
def self.ip_route_replace(cmd)
|
||||
run(:ip, 'route', 'replace', *cmd.strip.split(' '))
|
||||
end
|
||||
|
||||
def self.nft_json(cmd, **opts)
|
||||
o, e, s = run(:nft, '-j', *cmd.strip.split(' '), **opts)
|
||||
if s.success?
|
||||
[JSON.parse(o), e, s]
|
||||
else
|
||||
[{}, e, s]
|
||||
end
|
||||
end
|
||||
|
||||
def self.nft(script, **opts)
|
||||
run(:nft, '-f-', **opts, :stdin_data => script)
|
||||
end
|
||||
|
||||
def self.sysctl(cmd)
|
||||
run(:sysctl, *cmd.strip.split(' '))
|
||||
end
|
||||
|
||||
private_class_method def self.run(sym, *args, **opts)
|
||||
VNMNetwork::Command.no_shell(sym, *args, **opts)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
@ -53,9 +53,6 @@ module VNMMAD
|
||||
# Create the bridge.
|
||||
create_bridge(@nic)
|
||||
|
||||
# Setup transparent proxies.
|
||||
TProxy.setup_tproxy(@nic, :up)
|
||||
|
||||
# Check that no other vlans are connected to this bridge
|
||||
validate_vlan_id if @nic[:conf][:validate_vlan_id]
|
||||
|
||||
@ -134,9 +131,6 @@ module VNMMAD
|
||||
|
||||
@bridges[@nic[:bridge]].delete(@nic[:vlan_dev])
|
||||
|
||||
# Setup transparent proxies.
|
||||
TProxy.setup_tproxy(@nic, :down)
|
||||
|
||||
# Delete the bridge.
|
||||
OpenNebula.exec_and_log("#{command(:ip)} link delete"\
|
||||
" #{@nic[:bridge]}")
|
||||
|
||||
@ -33,7 +33,6 @@ require 'sg_driver'
|
||||
require 'vlan'
|
||||
require 'no_vlan'
|
||||
require 'scripts_common'
|
||||
require 'tproxy'
|
||||
|
||||
Dir[File.expand_path('vnmmad-load.d', File.dirname(__FILE__)) + "/*.rb"].each{ |f| require f }
|
||||
|
||||
@ -53,10 +52,7 @@ rescue
|
||||
:vlan_mtu => '1500',
|
||||
:ipset_maxelem => '65536',
|
||||
:keep_empty_bridge => false,
|
||||
:datastore_location => '/var/lib/one/datastores',
|
||||
:tproxy_debug_level => 2, # 0 = ERROR, 1 = WARNING, 2 = INFO, 3 = DEBUG
|
||||
:tproxy_process_owner => 'oneadmin',
|
||||
:tproxy => []
|
||||
:datastore_location => '/var/lib/one/datastores'
|
||||
}
|
||||
end
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user