From a9d43e9f3a4e5e8d2743bc412b829cfeb8b27120 Mon Sep 17 00:00:00 2001 From: sergio semedi Date: Wed, 16 Jan 2019 16:43:00 +0100 Subject: [PATCH 01/15] F #2320: migrate image gets proper format --- src/datastore_mad/remotes/vcenter/cp | 6 ------ src/oca/ruby/opennebula/marketplaceapp.rb | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/datastore_mad/remotes/vcenter/cp b/src/datastore_mad/remotes/vcenter/cp index abc273fcaa..9ec3fd4830 100755 --- a/src/datastore_mad/remotes/vcenter/cp +++ b/src/datastore_mad/remotes/vcenter/cp @@ -57,15 +57,9 @@ md5 = drv_action["/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5"] sha1 = drv_action["/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1"] nodecomp = drv_action["/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS"] limit_bw = drv_action["/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW"] -format = drv_action["IMAGE/TEMPLATE/FORMAT"] check_valid img_path, "img_path" -if format != 'vmdk' && format != 'iso' - one_img = VCenterDriver::VIHelper.one_item(OpenNebula::Image, id) - one_img.replace({'FORMAT' => 'vmdk'}) -end - # if image is already in a vCenter datastore return the path if img_path.start_with? "vcenter://" img_path = img_path.sub("vcenter://", "") diff --git a/src/oca/ruby/opennebula/marketplaceapp.rb b/src/oca/ruby/opennebula/marketplaceapp.rb index 88ff95a32e..36352cc2bb 100644 --- a/src/oca/ruby/opennebula/marketplaceapp.rb +++ b/src/oca/ruby/opennebula/marketplaceapp.rb @@ -193,6 +193,17 @@ module OpenNebula image = Image.new(Image.build_xml, @client) rc = image.allocate(tmpl, options[:dsid]) + ds = OpenNebula::Datastore.new_with_id(options[:dsid], @client) + image.info + ds.info + + xpath = 'TEMPLATE/DRIVER' + if ds[xpath] == 'vcenter' && self['FORMAT'] != 'iso' && self['FORMAT'] != 'vmdk' + image.replace({'FORMAT' => 'vmdk'}) + elsif ds[xpath] && ds[xpath] != 'vcenter' && self['FORMAT'] == 'vmdk' + image.replace({'FORMAT' => ds[xpath] }) + end + return { :image => [rc] } if OpenNebula.is_error?(rc) image_id = image.id From fad4a1c4543ccace743bdaa08f4ffa60ba43ab14 Mon Sep 17 00:00:00 2001 From: Jan Orel Date: Thu, 17 Jan 2019 13:34:37 +0100 Subject: [PATCH 02/15] B #1136 Avoid using disabled/off hosts --- src/datastore_mad/remotes/libfs.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/datastore_mad/remotes/libfs.sh b/src/datastore_mad/remotes/libfs.sh index ada2cbb5ad..8986bb9fdc 100644 --- a/src/datastore_mad/remotes/libfs.sh +++ b/src/datastore_mad/remotes/libfs.sh @@ -450,6 +450,31 @@ function check_restricted { echo 0 } +#------------------------------------------------------------------------------- +# Filter out hosts which are not ON +# @param $1 - space separated list of hosts +# @return - space separated list of hosts which are in ON state +#------------------------------------------------------------------------------- +function get_only_on_hosts { + INPUT_ARRAY=($1) + + ONEHOST_LIST_ON_CMD='onehost list --no-pager --csv --filter="STAT=on" --list=NAME,STAT' + ON_HOSTS_STR=$(eval "$ONEHOST_LIST_ON_CMD 2>/dev/null") + + if [[ $? = 0 ]]; then + ON_HOSTS_ARRAY=($( echo "$ON_HOSTS_STR" | $AWK -F, '{ if (NR>1) print $1 }')) + for A in "${INPUT_ARRAY[@]}"; do + for B in "${ON_HOSTS_ARRAY[@]}"; do + [[ $A == $B ]] && { echo $A; break; } + done + done + else + # onehost cmd failed, can't filter anything, better return unchanged + echo $1 + return 1 + fi +} + #------------------------------------------------------------------------------- # Gets the host to be used as bridge to talk to the storage system # Implements a round robin for the bridges @@ -458,6 +483,7 @@ function check_restricted { # @return host to be used as bridge #------------------------------------------------------------------------------- function get_destination_host { + BRIDGE_LIST=$(get_only_on_hosts "$BRIDGE_LIST") HOSTS_ARRAY=($BRIDGE_LIST) N_HOSTS=${#HOSTS_ARRAY[@]} From 24af2b77d264b2de2aef1a2070c5e6966ef2c534 Mon Sep 17 00:00:00 2001 From: sergio semedi Date: Fri, 18 Jan 2019 15:37:21 +0100 Subject: [PATCH 03/15] B #2828: vCenter template without any nic do not fail at deploy --- src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb b/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb index 020b8a164f..ebdc034361 100644 --- a/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb +++ b/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb @@ -1573,7 +1573,7 @@ class VirtualMachine < VCenterDriver::Template end # grab the last unitNumber to ensure the nic to be added at the end - @unic = @unic || get_vcenter_nics.map{|d| d.unitNumber}.max rescue 0 + @unic = @unic || get_vcenter_nics.map{|d| d.unitNumber}.max || 0 card_spec = { :key => 0, :deviceInfo => { From 3ed4ede17d8ab1da9b74e623523fbe3e7b2f38e8 Mon Sep 17 00:00:00 2001 From: Jan Orel Date: Fri, 18 Jan 2019 15:44:44 +0100 Subject: [PATCH 04/15] B #2629 Prevent race-condition when spawning multiple VMs --- src/mad/ruby/scripts_common.rb | 4 ++-- src/vnm_mad/remotes/ovswitch/OpenvSwitch.rb | 2 +- src/vnm_mad/remotes/vxlan/vxlan.rb | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mad/ruby/scripts_common.rb b/src/mad/ruby/scripts_common.rb index 29642ffd25..4381299a9b 100644 --- a/src/mad/ruby/scripts_common.rb +++ b/src/mad/ruby/scripts_common.rb @@ -75,11 +75,11 @@ module OpenNebula # Executes a command, if it fails returns error message and exits # If a second parameter is present it is used as the error message when # the command fails - def self.exec_and_log(command, message=nil) + def self.exec_and_log(command, message=nil, allowed_return_code=0) output=`#{command} 2>&1 1>/dev/null` code=$?.exitstatus - if code!=0 + if code!=0 && code!=allowed_return_code log_error "Command \"#{command}\" failed." log_error output if !message diff --git a/src/vnm_mad/remotes/ovswitch/OpenvSwitch.rb b/src/vnm_mad/remotes/ovswitch/OpenvSwitch.rb index 33ed662ff3..1bd32f253e 100644 --- a/src/vnm_mad/remotes/ovswitch/OpenvSwitch.rb +++ b/src/vnm_mad/remotes/ovswitch/OpenvSwitch.rb @@ -391,7 +391,7 @@ private def create_bridge return if @bridges.keys.include? @nic[:bridge] - OpenNebula.exec_and_log("#{command(:ovs_vsctl)} add-br #{@nic[:bridge]}") + OpenNebula.exec_and_log("#{command(:ovs_vsctl)} --may-exist add-br #{@nic[:bridge]}") set_bridge_options diff --git a/src/vnm_mad/remotes/vxlan/vxlan.rb b/src/vnm_mad/remotes/vxlan/vxlan.rb index a74f026d6c..1c67d467e4 100644 --- a/src/vnm_mad/remotes/vxlan/vxlan.rb +++ b/src/vnm_mad/remotes/vxlan/vxlan.rb @@ -65,9 +65,11 @@ module VXLAN ip_link_conf << "#{option} #{value} " end + # `ip link add ...` returns 2 when vxlan device already exists + # allow it to prevent race conditions OpenNebula.exec_and_log("#{command(:ip)} link add #{@nic[@attr_vlan_dev]}"\ " #{mtu} type vxlan id #{@nic[@attr_vlan_id]} #{group} #{ttl}"\ - " #{tep} #{ip_link_conf}") + " #{tep} #{ip_link_conf}", nil, 2) OpenNebula.exec_and_log("#{command(:ip)} link set #{@nic[@attr_vlan_dev]} up") end From 19f04965d486150cd1f5726581fe98bf94125bc4 Mon Sep 17 00:00:00 2001 From: Jan Orel Date: Fri, 18 Jan 2019 15:48:10 +0100 Subject: [PATCH 05/15] F #1012: update websockify to latest version (#2809) --- install.sh | 13 +- share/websockify/run | 5 + share/websockify/websockify | 1 - share/websockify/websockify/__init__.py | 2 + share/websockify/websockify/auth_plugins.py | 83 +++++++ share/websockify/websockify/token_plugins.py | 83 +++++++ .../websockify/{ => websockify}/websocket.py | 123 ++++++++-- .../{ => websockify}/websocketproxy.py | 212 +++++++++++++----- src/sunstone/OpenNebulaVNC.rb | 7 +- 9 files changed, 446 insertions(+), 83 deletions(-) create mode 100755 share/websockify/run delete mode 120000 share/websockify/websockify create mode 100644 share/websockify/websockify/__init__.py create mode 100644 share/websockify/websockify/auth_plugins.py create mode 100644 share/websockify/websockify/token_plugins.py rename share/websockify/{ => websockify}/websocket.py (89%) rename share/websockify/{ => websockify}/websocketproxy.py (72%) mode change 100644 => 100755 diff --git a/install.sh b/install.sh index 795fd40c86..a058ae33c5 100755 --- a/install.sh +++ b/install.sh @@ -219,6 +219,7 @@ fi SHARE_DIRS="$SHARE_LOCATION/examples \ $SHARE_LOCATION/websockify \ + $SHARE_LOCATION/websockify/websockify \ $SHARE_LOCATION/esx-fw-vnc \ $SHARE_LOCATION/oneprovision" @@ -477,7 +478,8 @@ INSTALL_FILES=( NETWORK_OVSWITCH_VXLAN_FILES:$VAR_LOCATION/remotes/vnm/ovswitch_vxlan NETWORK_VCENTER_FILES:$VAR_LOCATION/remotes/vnm/vcenter EXAMPLE_SHARE_FILES:$SHARE_LOCATION/examples - WEBSOCKIFY_SHARE_FILES:$SHARE_LOCATION/websockify + WEBSOCKIFY_SHARE_RUN_FILES:$SHARE_LOCATION/websockify + WEBSOCKIFY_SHARE_MODULE_FILES:$SHARE_LOCATION/websockify/websockify ESX_FW_VNC_SHARE_FILES:$SHARE_LOCATION/esx-fw-vnc INSTALL_GEMS_SHARE_FILES:$SHARE_LOCATION ONETOKEN_SHARE_FILE:$SHARE_LOCATION @@ -1530,9 +1532,12 @@ EXAMPLE_SHARE_FILES="share/examples/vm.template \ # Files required to interact with the websockify server #------------------------------------------------------------------------------- -WEBSOCKIFY_SHARE_FILES="share/websockify/websocketproxy.py \ - share/websockify/websocket.py \ - share/websockify/websockify" +WEBSOCKIFY_SHARE_RUN_FILES="share/websockify/run" +WEBSOCKIFY_SHARE_MODULE_FILES="share/websockify/websockify/__init__.py \ + share/websockify/websockify/auth_plugins.py \ + share/websockify/websockify/token_plugins.py \ + share/websockify/websockify/websocket.py \ + share/websockify/websockify/websocketproxy.py" #------------------------------------------------------------------------------- # Installation packages for ESX hosts to enable VNC ports diff --git a/share/websockify/run b/share/websockify/run new file mode 100755 index 0000000000..9ad217c04c --- /dev/null +++ b/share/websockify/run @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +import websockify + +websockify.websocketproxy.websockify_init() diff --git a/share/websockify/websockify b/share/websockify/websockify deleted file mode 120000 index 9558147b09..0000000000 --- a/share/websockify/websockify +++ /dev/null @@ -1 +0,0 @@ -websocketproxy.py \ No newline at end of file diff --git a/share/websockify/websockify/__init__.py b/share/websockify/websockify/__init__.py new file mode 100644 index 0000000000..37a6f47b45 --- /dev/null +++ b/share/websockify/websockify/__init__.py @@ -0,0 +1,2 @@ +from websockify.websocket import * +from websockify.websocketproxy import * diff --git a/share/websockify/websockify/auth_plugins.py b/share/websockify/websockify/auth_plugins.py new file mode 100644 index 0000000000..93f1385504 --- /dev/null +++ b/share/websockify/websockify/auth_plugins.py @@ -0,0 +1,83 @@ +class BasePlugin(object): + def __init__(self, src=None): + self.source = src + + def authenticate(self, headers, target_host, target_port): + pass + + +class AuthenticationError(Exception): + def __init__(self, log_msg=None, response_code=403, response_headers={}, response_msg=None): + self.code = response_code + self.headers = response_headers + self.msg = response_msg + + if log_msg is None: + log_msg = response_msg + + super(AuthenticationError, self).__init__('%s %s' % (self.code, log_msg)) + + +class InvalidOriginError(AuthenticationError): + def __init__(self, expected, actual): + self.expected_origin = expected + self.actual_origin = actual + + super(InvalidOriginError, self).__init__( + response_msg='Invalid Origin', + log_msg="Invalid Origin Header: Expected one of " + "%s, got '%s'" % (expected, actual)) + + +class BasicHTTPAuth(object): + """Verifies Basic Auth headers. Specify src as username:password""" + + def __init__(self, src=None): + self.src = src + + def authenticate(self, headers, target_host, target_port): + import base64 + auth_header = headers.get('Authorization') + if auth_header: + if not auth_header.startswith('Basic '): + raise AuthenticationError(response_code=403) + + try: + user_pass_raw = base64.b64decode(auth_header[6:]) + except TypeError: + raise AuthenticationError(response_code=403) + + try: + # http://stackoverflow.com/questions/7242316/what-encoding-should-i-use-for-http-basic-authentication + user_pass_as_text = user_pass_raw.decode('ISO-8859-1') + except UnicodeDecodeError: + raise AuthenticationError(response_code=403) + + user_pass = user_pass_as_text.split(':', 1) + if len(user_pass) != 2: + raise AuthenticationError(response_code=403) + + if not self.validate_creds(*user_pass): + raise AuthenticationError(response_code=403) + + else: + raise AuthenticationError(response_code=401, + response_headers={'WWW-Authenticate': 'Basic realm="Websockify"'}) + + def validate_creds(self, username, password): + if '%s:%s' % (username, password) == self.src: + return True + else: + return False + +class ExpectOrigin(object): + def __init__(self, src=None): + if src is None: + self.source = [] + else: + self.source = src.split() + + def authenticate(self, headers, target_host, target_port): + origin = headers.get('Origin', None) + if origin is None or origin not in self.source: + raise InvalidOriginError(expected=self.source, actual=origin) diff --git a/share/websockify/websockify/token_plugins.py b/share/websockify/websockify/token_plugins.py new file mode 100644 index 0000000000..92494eb03f --- /dev/null +++ b/share/websockify/websockify/token_plugins.py @@ -0,0 +1,83 @@ +import os + +class BasePlugin(object): + def __init__(self, src): + self.source = src + + def lookup(self, token): + return None + + +class ReadOnlyTokenFile(BasePlugin): + # source is a token file with lines like + # token: host:port + # or a directory of such files + def __init__(self, *args, **kwargs): + super(ReadOnlyTokenFile, self).__init__(*args, **kwargs) + self._targets = None + + def _load_targets(self): + if os.path.isdir(self.source): + cfg_files = [os.path.join(self.source, f) for + f in os.listdir(self.source)] + else: + cfg_files = [self.source] + + self._targets = {} + for f in cfg_files: + for line in [l.strip() for l in open(f).readlines()]: + if line and not line.startswith('#'): + tok, target = line.split(': ') + self._targets[tok] = target.strip().rsplit(':', 1) + + def lookup(self, token): + if self._targets is None: + self._load_targets() + + if token in self._targets: + return self._targets[token] + else: + return None + + +# the above one is probably more efficient, but this one is +# more backwards compatible (although in most cases +# ReadOnlyTokenFile should suffice) +class TokenFile(ReadOnlyTokenFile): + # source is a token file with lines like + # token: host:port + # or a directory of such files + def lookup(self, token): + self._load_targets() + + return super(TokenFile, self).lookup(token) + + +class BaseTokenAPI(BasePlugin): + # source is a url with a '%s' in it where the token + # should go + + # we import things on demand so that other plugins + # in this file can be used w/o unecessary dependencies + + def process_result(self, resp): + return resp.text.split(':') + + def lookup(self, token): + import requests + + resp = requests.get(self.source % token) + + if resp.ok: + return self.process_result(resp) + else: + return None + + +class JSONTokenApi(BaseTokenAPI): + # source is a url with a '%s' in it where the token + # should go + + def process_result(self, resp): + resp_json = resp.json() + return (resp_json['host'], resp_json['port']) diff --git a/share/websockify/websocket.py b/share/websockify/websockify/websocket.py similarity index 89% rename from share/websockify/websocket.py rename to share/websockify/websockify/websocket.py index d161f648c5..274a0047fc 100644 --- a/share/websockify/websocket.py +++ b/share/websockify/websockify/websocket.py @@ -104,6 +104,8 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): self.handler_id = getattr(server, "handler_id", False) self.file_only = getattr(server, "file_only", False) self.traffic = getattr(server, "traffic", False) + self.auto_pong = getattr(server, "auto_pong", False) + self.strict_mode = getattr(server, "strict_mode", True) self.logger = getattr(server, "logger", None) if self.logger is None: @@ -111,6 +113,9 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): SimpleHTTPRequestHandler.__init__(self, req, addr, server) + def log_message(self, format, *args): + self.logger.info("%s - - [%s] %s" % (self.address_string(), self.log_date_time_string(), format % args)) + @staticmethod def unmask(buf, hlen, plen): pstart = hlen + 4 @@ -118,20 +123,24 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): if numpy: b = c = s2b('') if plen >= 4: - mask = numpy.frombuffer(buf, dtype=numpy.dtype('') + mask = numpy.frombuffer(buf, dtype, offset=hlen, count=1) + data = numpy.frombuffer(buf, dtype, offset=pstart, + count=int(plen / 4)) #b = numpy.bitwise_xor(data, mask).data b = numpy.bitwise_xor(data, mask).tostring() if plen % 4: #self.msg("Partial unmask") - mask = numpy.frombuffer(buf, dtype=numpy.dtype('B'), - offset=hlen, count=(plen % 4)) - data = numpy.frombuffer(buf, dtype=numpy.dtype('B'), - offset=pend - (plen % 4), + dtype=numpy.dtype('B') + if sys.byteorder == 'big': + dtype = dtype.newbyteorder('>') + mask = numpy.frombuffer(buf, dtype, offset=hlen, count=(plen % 4)) + data = numpy.frombuffer(buf, dtype, + offset=pend - (plen % 4), count=(plen % 4)) c = numpy.bitwise_xor(data, mask).tostring() return b + c else: @@ -172,7 +181,7 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): return header + buf, len(header), 0 @staticmethod - def decode_hybi(buf, base64=False, logger=None): + def decode_hybi(buf, base64=False, logger=None, strict=True): """ Decode HyBi style WebSocket packets. Returns: {'fin' : 0_or_1, @@ -238,6 +247,10 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): f['length']) else: logger.debug("Unmasked frame: %s" % repr(buf)) + + if strict: + raise WebSocketRequestHandler.CClose(1002, "The client sent an unmasked frame.") + f['payload'] = buf[(f['hlen'] + f['masked'] * 4):full_len] if base64 and f['opcode'] in [1, 2]: @@ -346,7 +359,8 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): while buf: frame = self.decode_hybi(buf, base64=self.base64, - logger=self.logger) + logger=self.logger, + strict=self.strict_mode) #self.msg("Received buf: %s, frame: %s", repr(buf), frame) if frame['payload'] == None: @@ -360,6 +374,15 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): closed = {'code': frame['close_code'], 'reason': frame['close_reason']} break + elif self.auto_pong and frame['opcode'] == 0x9: # ping + self.print_traffic("} ping %s\n" % + repr(frame['payload'])) + self.send_pong(frame['payload']) + return [], False + elif frame['opcode'] == 0xA: # pong + self.print_traffic("} pong %s\n" % + repr(frame['payload'])) + return [], False self.print_traffic("}") @@ -388,10 +411,20 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): def send_close(self, code=1000, reason=''): """ Send a WebSocket orderly close frame. """ - msg = pack(">H%ds" % len(reason), code, reason) + msg = pack(">H%ds" % len(reason), code, s2b(reason)) buf, h, t = self.encode_hybi(msg, opcode=0x08, base64=False) self.request.send(buf) + def send_pong(self, data=''): + """ Send a WebSocket pong frame. """ + buf, h, t = self.encode_hybi(s2b(data), opcode=0x0A, base64=False) + self.request.send(buf) + + def send_ping(self, data=''): + """ Send a WebSocket ping frame. """ + buf, h, t = self.encode_hybi(s2b(data), opcode=0x09, base64=False) + self.request.send(buf) + def do_websocket_handshake(self): h = self.headers @@ -444,9 +477,13 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): """Upgrade a connection to Websocket, if requested. If this succeeds, new_websocket_client() will be called. Otherwise, False is returned. """ + if (self.headers.get('upgrade') and self.headers.get('upgrade').lower() == 'websocket'): + # ensure connection is authorized, and determine the target + self.validate_connection() + if not self.do_websocket_handshake(): return False @@ -519,6 +556,10 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler): """ Do something with a WebSockets client connection. """ raise Exception("WebSocketRequestHandler.new_websocket_client() must be overloaded") + def validate_connection(self): + """ Ensure that the connection is a valid connection, and set the target. """ + pass + def do_HEAD(self): if self.only_upgrade: self.send_error(405, "Method Not Allowed") @@ -567,7 +608,7 @@ class WebSocketServer(object): file_only=False, run_once=False, timeout=0, idle_timeout=0, traffic=False, tcp_keepalive=True, tcp_keepcnt=None, tcp_keepidle=None, - tcp_keepintvl=None): + tcp_keepintvl=None, auto_pong=False, strict_mode=True): # settings self.RequestHandlerClass = RequestHandlerClass @@ -581,6 +622,8 @@ class WebSocketServer(object): self.timeout = timeout self.idle_timeout = idle_timeout self.traffic = traffic + self.file_only = file_only + self.strict_mode = strict_mode self.launch_time = time.time() self.ws_connection = False @@ -592,6 +635,7 @@ class WebSocketServer(object): self.tcp_keepidle = tcp_keepidle self.tcp_keepintvl = tcp_keepintvl + self.auto_pong = auto_pong # Make paths settings absolute self.cert = os.path.abspath(cert) self.key = self.web = self.record = '' @@ -618,7 +662,10 @@ class WebSocketServer(object): self.listen_host, self.listen_port) self.msg(" - Flash security policy server") if self.web: - self.msg(" - Web server. Web root: %s", self.web) + if self.file_only: + self.msg(" - Web server (no directory listings). Web root: %s", self.web) + else: + self.msg(" - Web server. Web root: %s", self.web) if ssl: if os.path.exists(self.cert): self.msg(" - SSL/TLS support") @@ -701,6 +748,10 @@ class WebSocketServer(object): @staticmethod def daemonize(keepfd=None, chdir='/'): + + if keepfd is None: + keepfd = [] + os.umask(0) if chdir: os.chdir(chdir) @@ -723,7 +774,7 @@ class WebSocketServer(object): if maxfd == resource.RLIM_INFINITY: maxfd = 256 for fd in reversed(range(maxfd)): try: - if fd != keepfd: + if fd not in keepfd: os.close(fd) except OSError: _, exc, _ = sys.exc_info() @@ -753,7 +804,7 @@ class WebSocketServer(object): """ ready = select.select([sock], [], [], 3)[0] - + if not ready: raise self.EClose("ignoring socket not ready") # Peek, but do not read the data so that we have a opportunity @@ -761,7 +812,7 @@ class WebSocketServer(object): handshake = sock.recv(1024, socket.MSG_PEEK) #self.msg("Handshake [%s]" % handshake) - if handshake == "": + if not handshake: raise self.EClose("ignoring empty handshake") elif handshake.startswith(s2b("")): @@ -844,11 +895,14 @@ class WebSocketServer(object): raise self.Terminate() def multiprocessing_SIGCHLD(self, sig, stack): - self.vmsg('Reaping zombies, active child count is %s', len(multiprocessing.active_children())) + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers + multiprocessing.active_children() def fallback_SIGCHLD(self, sig, stack): # Reap zombies when using os.fork() (python 2.4) - self.vmsg("Got SIGCHLD, reaping zombies") + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers try: result = os.waitpid(-1, os.WNOHANG) while result[0]: @@ -858,16 +912,18 @@ class WebSocketServer(object): pass def do_SIGINT(self, sig, stack): - self.msg("Got SIGINT, exiting") + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers self.terminate() def do_SIGTERM(self, sig, stack): - self.msg("Got SIGTERM, exiting") + # TODO: figure out a way to actually log this information without + # calling `log` in the signal handlers self.terminate() def top_new_client(self, startsock, address): """ Do something with a WebSockets client connection. """ - # handler process + # handler process client = None try: try: @@ -890,6 +946,18 @@ class WebSocketServer(object): # Original socket closed by caller client.close() + def get_log_fd(self): + """ + Get file descriptors for the loggers. + They should not be closed when the process is forked. + """ + descriptors = [] + for handler in self.logger.parent.handlers: + if isinstance(handler, logging.FileHandler): + descriptors.append(handler.stream.fileno()) + + return descriptors + def start_server(self): """ Daemonize if requested. Listen for for connections. Run @@ -905,7 +973,9 @@ class WebSocketServer(object): tcp_keepintvl=self.tcp_keepintvl) if self.daemon: - self.daemonize(keepfd=lsock.fileno(), chdir=self.web) + keepfd = self.get_log_fd() + keepfd.append(lsock.fileno()) + self.daemonize(keepfd=keepfd, chdir=self.web) self.started() # Some things need to happen after daemonizing @@ -1009,8 +1079,17 @@ class WebSocketServer(object): except (self.Terminate, SystemExit, KeyboardInterrupt): self.msg("In exit") + # terminate all child processes + if multiprocessing and not self.run_once: + children = multiprocessing.active_children() + + for child in children: + self.msg("Terminating child %s" % child.pid) + child.terminate() + break except Exception: + exc = sys.exc_info()[1] self.msg("handler exception: %s", str(exc)) self.vmsg("exception", exc_info=True) diff --git a/share/websockify/websocketproxy.py b/share/websockify/websockify/websocketproxy.py old mode 100644 new mode 100755 similarity index 72% rename from share/websockify/websocketproxy.py rename to share/websockify/websockify/websocketproxy.py index 7b3ec111df..81e2248f21 --- a/share/websockify/websocketproxy.py +++ b/share/websockify/websockify/websocketproxy.py @@ -11,13 +11,14 @@ as taken from http://docs.python.org/dev/library/ssl.html#certificates ''' -import signal, socket, optparse, time, os, sys, subprocess, logging +import signal, socket, optparse, time, os, sys, subprocess, logging, errno try: from socketserver import ForkingMixIn except: from SocketServer import ForkingMixIn try: from http.server import HTTPServer except: from BaseHTTPServer import HTTPServer -from select import select -import websocket +import select +from websockify import websocket +from websockify import auth_plugins as auth try: from urllib.parse import parse_qs, urlparse except: @@ -38,14 +39,33 @@ Traffic Legend: <. - Client send partial """ + def send_auth_error(self, ex): + self.send_response(ex.code, ex.msg) + self.send_header('Content-Type', 'text/html') + for name, val in ex.headers.items(): + self.send_header(name, val) + + self.end_headers() + + def validate_connection(self): + if self.server.token_plugin: + (self.server.target_host, self.server.target_port) = self.get_target(self.server.token_plugin, self.path) + + if self.server.auth_plugin: + try: + self.server.auth_plugin.authenticate( + headers=self.headers, target_host=self.server.target_host, + target_port=self.server.target_port) + except auth.AuthenticationError: + ex = sys.exc_info()[1] + self.send_auth_error(ex) + raise + def new_websocket_client(self): """ Called after a new WebSocket connection has been established. """ - # Checks if we receive a token, and look - # for a valid target for it then - if self.server.target_cfg: - (self.server.target_host, self.server.target_port) = self.get_target(self.server.target_cfg, self.path) + # Checking for a token is done in validate_connection() # Connect to the target if self.server.wrap_cmd: @@ -73,15 +93,15 @@ Traffic Legend: if tsock: tsock.shutdown(socket.SHUT_RDWR) tsock.close() - if self.verbose: + if self.verbose: self.log_message("%s:%s: Closed target", self.server.target_host, self.server.target_port) raise - def get_target(self, target_cfg, path): + def get_target(self, target_plugin, path): """ - Parses the path, extracts a token, and looks for a valid - target for that token in the configuration file(s). Sets + Parses the path, extracts a token, and looks up a target + for that token using the token plugin. Sets target_host and target_port if successful """ # The files in targets contain the lines @@ -90,32 +110,17 @@ Traffic Legend: # Extract the token parameter from url args = parse_qs(urlparse(path)[4]) # 4 is the query from url - if not args.has_key('token') or not len(args['token']): - raise self.EClose("Token not present") + if not 'token' in args or not len(args['token']): + raise self.server.EClose("Token not present") token = args['token'][0].rstrip('\n') - # target_cfg can be a single config file or directory of - # config files - if os.path.isdir(target_cfg): - cfg_files = [os.path.join(target_cfg, f) - for f in os.listdir(target_cfg)] + result_pair = target_plugin.lookup(token) + + if result_pair is not None: + return result_pair else: - cfg_files = [target_cfg] - - targets = {} - for f in cfg_files: - for line in [l.strip() for l in file(f).readlines()]: - if line and not line.startswith('#'): - ttoken, target = line.split(': ') - targets[ttoken] = target.strip() - - self.vmsg("Target config: %s" % repr(targets)) - - if targets.has_key(token): - return targets[token].split(':') - else: - raise self.EClose("Token '%s' not found" % token) + raise self.server.EClose("Token '%s' not found" % token) def do_proxy(self, target): """ @@ -126,12 +131,37 @@ Traffic Legend: tqueue = [] rlist = [self.request, target] + if self.server.heartbeat: + now = time.time() + self.heartbeat = now + self.server.heartbeat + else: + self.heartbeat = None + while True: wlist = [] + if self.heartbeat is not None: + now = time.time() + if now > self.heartbeat: + self.heartbeat = now + self.server.heartbeat + self.send_ping() + if tqueue: wlist.append(target) if cqueue or c_pend: wlist.append(self.request) - ins, outs, excepts = select(rlist, wlist, [], 1) + try: + ins, outs, excepts = select.select(rlist, wlist, [], 1) + except (select.error, OSError): + exc = sys.exc_info()[1] + if hasattr(exc, 'errno'): + err = exc.errno + else: + err = exc[0] + + if err != errno.EINTR: + raise + else: + continue + if excepts: raise Exception("Socket exception") if self.request in outs: @@ -147,7 +177,7 @@ Traffic Legend: if closed: # TODO: What about blocking on client socket? - if self.verbose: + if self.verbose: self.log_message("%s:%s: Client closed connection", self.server.target_host, self.server.target_port) raise self.CClose(closed['code'], closed['reason']) @@ -195,7 +225,11 @@ class WebSocketProxy(websocket.WebSocketServer): self.wrap_mode = kwargs.pop('wrap_mode', None) self.unix_target = kwargs.pop('unix_target', None) self.ssl_target = kwargs.pop('ssl_target', None) - self.target_cfg = kwargs.pop('target_cfg', None) + self.heartbeat = kwargs.pop('heartbeat', None) + + self.token_plugin = kwargs.pop('token_plugin', None) + self.auth_plugin = kwargs.pop('auth_plugin', None) + # Last 3 timestamps command was run self.wrap_times = [0, 0, 0] @@ -251,9 +285,9 @@ class WebSocketProxy(websocket.WebSocketServer): else: dst_string = "%s:%s" % (self.target_host, self.target_port) - if self.target_cfg: - msg = " - proxying from %s:%s to targets in %s" % ( - self.listen_host, self.listen_port, self.target_cfg) + if self.token_plugin: + msg = " - proxying from %s:%s to targets generated by %s" % ( + self.listen_host, self.listen_port, type(self.token_plugin).__name__) else: msg = " - proxying from %s:%s to %s" % ( self.listen_host, self.listen_port, dst_string) @@ -352,20 +386,69 @@ def websockify_init(): parser.add_option("--prefer-ipv6", "-6", action="store_true", dest="source_is_ipv6", help="prefer IPv6 when resolving source_addr") + parser.add_option("--libserver", action="store_true", + help="use Python library SocketServer engine") parser.add_option("--target-config", metavar="FILE", dest="target_cfg", help="Configuration file containing valid targets " "in the form 'token: host:port' or, alternatively, a " - "directory containing configuration files of this form") - parser.add_option("--libserver", action="store_true", - help="use Python library SocketServer engine") + "directory containing configuration files of this form " + "(DEPRECATED: use `--token-plugin TokenFile --token-source " + " path/to/token/file` instead)") + parser.add_option("--token-plugin", default=None, metavar="PLUGIN", + help="use the given Python class to process tokens " + "into host:port pairs") + parser.add_option("--token-source", default=None, metavar="ARG", + help="an argument to be passed to the token plugin" + "on instantiation") + parser.add_option("--auth-plugin", default=None, metavar="PLUGIN", + help="use the given Python class to determine if " + "a connection is allowed") + parser.add_option("--auth-source", default=None, metavar="ARG", + help="an argument to be passed to the auth plugin" + "on instantiation") + parser.add_option("--auto-pong", action="store_true", + help="Automatically respond to ping frames with a pong") + parser.add_option("--heartbeat", type=int, default=0, + help="send a ping to the client every HEARTBEAT seconds") + parser.add_option("--log-file", metavar="FILE", + dest="log_file", + help="File where logs will be saved") + + (opts, args) = parser.parse_args() + if opts.log_file: + opts.log_file = os.path.abspath(opts.log_file) + handler = logging.FileHandler(opts.log_file) + handler.setLevel(logging.DEBUG) + handler.setFormatter(logging.Formatter("%(message)s")) + logging.getLogger(WebSocketProxy.log_prefix).addHandler(handler) + + del opts.log_file + if opts.verbose: logging.getLogger(WebSocketProxy.log_prefix).setLevel(logging.DEBUG) + if opts.token_source and not opts.token_plugin: + parser.error("You must use --token-plugin to use --token-source") + + if opts.auth_source and not opts.auth_plugin: + parser.error("You must use --auth-plugin to use --auth-source") + + + # Transform to absolute path as daemon may chdir + if opts.target_cfg: + opts.target_cfg = os.path.abspath(opts.target_cfg) + + if opts.target_cfg: + opts.token_plugin = 'TokenFile' + opts.token_source = opts.target_cfg + + del opts.target_cfg + # Sanity checks - if len(args) < 2 and not (opts.target_cfg or opts.unix_target): + if len(args) < 2 and not (opts.token_plugin or opts.unix_target): parser.error("Too few arguments") if sys.argv.count('--'): opts.wrap_cmd = args[1:] @@ -390,7 +473,7 @@ def websockify_init(): try: opts.listen_port = int(opts.listen_port) except: parser.error("Error parsing listen port") - if opts.wrap_cmd or opts.unix_target or opts.target_cfg: + if opts.wrap_cmd or opts.unix_target or opts.token_plugin: opts.target_host = None opts.target_port = None else: @@ -402,9 +485,32 @@ def websockify_init(): try: opts.target_port = int(opts.target_port) except: parser.error("Error parsing target port") - # Transform to absolute path as daemon may chdir - if opts.target_cfg: - opts.target_cfg = os.path.abspath(opts.target_cfg) + if opts.token_plugin is not None: + if '.' not in opts.token_plugin: + opts.token_plugin = ( + 'websockify.token_plugins.%s' % opts.token_plugin) + + token_plugin_module, token_plugin_cls = opts.token_plugin.rsplit('.', 1) + + __import__(token_plugin_module) + token_plugin_cls = getattr(sys.modules[token_plugin_module], token_plugin_cls) + + opts.token_plugin = token_plugin_cls(opts.token_source) + + del opts.token_source + + if opts.auth_plugin is not None: + if '.' not in opts.auth_plugin: + opts.auth_plugin = 'websockify.auth_plugins.%s' % opts.auth_plugin + + auth_plugin_module, auth_plugin_cls = opts.auth_plugin.rsplit('.', 1) + + __import__(auth_plugin_module) + auth_plugin_cls = getattr(sys.modules[auth_plugin_module], auth_plugin_cls) + + opts.auth_plugin = auth_plugin_cls(opts.auth_source) + + del opts.auth_source # Create and start the WebSockets proxy libserver = opts.libserver @@ -433,9 +539,13 @@ class LibProxyServer(ForkingMixIn, HTTPServer): self.wrap_mode = kwargs.pop('wrap_mode', None) self.unix_target = kwargs.pop('unix_target', None) self.ssl_target = kwargs.pop('ssl_target', None) - self.target_cfg = kwargs.pop('target_cfg', None) + self.token_plugin = kwargs.pop('token_plugin', None) + self.auth_plugin = kwargs.pop('auth_plugin', None) + self.heartbeat = kwargs.pop('heartbeat', None) + + self.token_plugin = None + self.auth_plugin = None self.daemon = False - self.target_cfg = None # Server configuration listen_host = kwargs.pop('listen_host', '') @@ -456,8 +566,8 @@ class LibProxyServer(ForkingMixIn, HTTPServer): if web: os.chdir(web) - - HTTPServer.__init__(self, (listen_host, listen_port), + + HTTPServer.__init__(self, (listen_host, listen_port), RequestHandlerClass) diff --git a/src/sunstone/OpenNebulaVNC.rb b/src/sunstone/OpenNebulaVNC.rb index f3908f6b2e..120f2a1745 100644 --- a/src/sunstone/OpenNebulaVNC.rb +++ b/src/sunstone/OpenNebulaVNC.rb @@ -109,7 +109,7 @@ class OpenNebulaVNC @pipe = nil @token_folder = File.join(VAR_LOCATION, opts[:token_folder_name]) - @proxy_path = File.join(SHARE_LOCATION, "websockify/websocketproxy.py") + @proxy_path = File.join(SHARE_LOCATION, "websockify/run") @proxy_port = config[:vnc_proxy_port] @proxy_ipv6 = config[:vnc_proxy_ipv6] @@ -151,10 +151,7 @@ class OpenNebulaVNC proxy_options << " -6" end - system("which python2 >/dev/null 2>&1") - python = $?.success? ? "python2" : "python" - - cmd ="#{python} #{@proxy_path} #{proxy_options} #{@proxy_port}" + cmd ="python #{@proxy_path} #{proxy_options} #{@proxy_port}" begin @logger.info { "Starting VNC proxy: #{cmd}" } From c5a89d57dec25658db1e8c1d6cfd3c23ee89f132 Mon Sep 17 00:00:00 2001 From: Jan Orel Date: Fri, 18 Jan 2019 15:49:43 +0100 Subject: [PATCH 06/15] development: PyOne make dist target phony, update gitignore (#2815) --- .gitignore | 2 +- src/oca/python/Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 4dc9354b9a..8a549e71de 100644 --- a/.gitignore +++ b/.gitignore @@ -28,7 +28,7 @@ src/vmm_mad/remotes/lxd/tests/ src/oca/python/pyone/bindings src/oca/python/build/ src/oca/python/dist/ -src/oca/python/opennebula.egg-info/ +src/oca/python/pyone.egg-info/ src/oca/python/doc/ src/docker_machine/pkg diff --git a/src/oca/python/Makefile b/src/oca/python/Makefile index cbf243391d..23bedb6816 100644 --- a/src/oca/python/Makefile +++ b/src/oca/python/Makefile @@ -14,7 +14,7 @@ # limitations under the License. # Use full path to ensure virtualenv compatibility -PYTHON = $(shell which python) +PYTHON = $(shell which python) GDS = $(shell which generateDS) PWD = $(shell pwd) @@ -35,7 +35,7 @@ pyone/bindings/__init__.py pyone/bindings/supbind.py: $(schemas) sed -i "s/import sys/import sys\nfrom pyone.util import TemplatedType/" pyone/bindings/__init__.py sed -i "s/(supermod\./(TemplatedType, supermod\./g" pyone/bindings/__init__.py -.PHONY: clean +.PHONY: clean dist clean: rm -rf build dist pyone/bindings *.egg-info doc From 948f929f6a431b7be248ce185f4d8d04340b3f50 Mon Sep 17 00:00:00 2001 From: Jan Orel Date: Thu, 17 Jan 2019 17:47:19 +0100 Subject: [PATCH 07/15] B #1136 Filter out only hosts which are off, disabled or error * There still might be hosts which are part of Ceph cluster but not OpenNebula * The host might be added afterwards --- src/datastore_mad/remotes/libfs.sh | 36 ++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/datastore_mad/remotes/libfs.sh b/src/datastore_mad/remotes/libfs.sh index 8986bb9fdc..eb10478907 100644 --- a/src/datastore_mad/remotes/libfs.sh +++ b/src/datastore_mad/remotes/libfs.sh @@ -451,22 +451,24 @@ function check_restricted { } #------------------------------------------------------------------------------- -# Filter out hosts which are not ON +# Filter out hosts which are OFF, ERROR or DISABLED # @param $1 - space separated list of hosts -# @return - space separated list of hosts which are in ON state +# @return - space separated list of hosts which are not in OFF, ERROR or +# DISABLED sate #------------------------------------------------------------------------------- -function get_only_on_hosts { - INPUT_ARRAY=($1) +function remove_off_hosts { + ALL_HOSTS_ARRAY=($1) + OFF_HOSTS_STR=$(onehost list --no-pager --csv \ + --filter="STAT=off,STAT=err,STAT=dsbl" --list=NAME,STAT 2>/dev/null) - ONEHOST_LIST_ON_CMD='onehost list --no-pager --csv --filter="STAT=on" --list=NAME,STAT' - ON_HOSTS_STR=$(eval "$ONEHOST_LIST_ON_CMD 2>/dev/null") - - if [[ $? = 0 ]]; then - ON_HOSTS_ARRAY=($( echo "$ON_HOSTS_STR" | $AWK -F, '{ if (NR>1) print $1 }')) - for A in "${INPUT_ARRAY[@]}"; do - for B in "${ON_HOSTS_ARRAY[@]}"; do - [[ $A == $B ]] && { echo $A; break; } + if [ $? -eq 0 ]; then + OFF_HOSTS_ARRAY=($( echo "$OFF_HOSTS_STR" | awk -F, '{ if (NR>1) print $1 }')) + for HOST in "${ALL_HOSTS_ARRAY[@]}"; do + OFF=false + for OFF_HOST in "${OFF_HOSTS_ARRAY[@]}"; do + [ $HOST = $OFF_HOST ] && { OFF=true; break; } done + $OFF || echo -ne "$HOST " done else # onehost cmd failed, can't filter anything, better return unchanged @@ -483,8 +485,14 @@ function get_only_on_hosts { # @return host to be used as bridge #------------------------------------------------------------------------------- function get_destination_host { - BRIDGE_LIST=$(get_only_on_hosts "$BRIDGE_LIST") - HOSTS_ARRAY=($BRIDGE_LIST) + REDUCED_LIST=$(remove_off_hosts "$BRIDGE_LIST") + + if [ -z "$REDUCED_LIST" -a -n "$BRIDGE_LIST" ]; then + error_message "All hosts from 'BRIDGE_LIST' are offline, error or disabled" + exit -1 + fi + + HOSTS_ARRAY=($REDUCED_LIST) N_HOSTS=${#HOSTS_ARRAY[@]} if [ -n "$1" ]; then From 65883dab983dd5af0216af5562db8da4d83fb856 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Sun, 20 Jan 2019 14:38:41 +0100 Subject: [PATCH 08/15] development: Enable by default configuration the shared mode for ceph --- share/etc/oned.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/etc/oned.conf b/share/etc/oned.conf index c1055fafc5..1c2d86bb1d 100644 --- a/share/etc/oned.conf +++ b/share/etc/oned.conf @@ -1307,7 +1307,7 @@ TM_MAD_CONF = [ TM_MAD_CONF = [ NAME = "ceph", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES", DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="mixed", - TM_MAD_SYSTEM = "ssh", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM", + TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM", DISK_TYPE_SSH = "FILE", TM_MAD_SYSTEM = "shared", LN_TARGET_SHARED = "NONE", CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "RBD" ] From ffdbf0bab1f45f99822718bc993539ac197fb831 Mon Sep 17 00:00:00 2001 From: Alejandro Huertas Herrero Date: Mon, 21 Jan 2019 12:46:31 +0100 Subject: [PATCH 09/15] B #2820: Fix CPU_MODEL can't be changed. (#2822) --- src/cli/onevm | 13 +++++++------ src/sunstone/public/app/tabs/vms-tab/panels/conf.js | 2 +- src/vm/VirtualMachine.cc | 7 +++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/cli/onevm b/src/cli/onevm index 7b62b22164..a2a0c04dc9 100755 --- a/src/cli/onevm +++ b/src/cli/onevm @@ -1128,11 +1128,12 @@ CommandParser::CmdParser.new(ARGV) do This command accepts a template file or opens an editor, the full list of configuration attributes are: - OS = ["ARCH", "MACHINE", "KERNEL", "INITRD", "BOOTLOADER", "BOOT"] - FEATURES = ["ACPI", "PAE", "APIC", "LOCALTIME", "HYPERV", "GUEST_AGENT"] - INPUT = ["TYPE", "BUS"] - GRAPHICS = ["TYPE", "LISTEN", "PASSWD", "KEYMAP" ] - RAW = ["DATA", "DATA_VMX", "TYPE"] + OS = ["ARCH", "MACHINE", "KERNEL", "INITRD", "BOOTLOADER", "BOOT"] + FEATURES = ["ACPI", "PAE", "APIC", "LOCALTIME", "HYPERV", "GUEST_AGENT"] + INPUT = ["TYPE", "BUS"] + GRAPHICS = ["TYPE", "LISTEN", "PASSWD", "KEYMAP" ] + RAW = ["DATA", "DATA_VMX", "TYPE"] + CPU_MODEL = ["MODEL"] CONTEXT (any value, **variable substitution will be made**) EOT @@ -1153,7 +1154,7 @@ CommandParser::CmdParser.new(ARGV) do template = vm.template_like_str('TEMPLATE', true, 'OS | FEATURES | INPUT | '\ - 'GRAPHICS | RAW | CONTEXT') + 'GRAPHICS | RAW | CONTEXT | CPU_MODEL') template = OpenNebulaHelper.editor_input(template) end diff --git a/src/sunstone/public/app/tabs/vms-tab/panels/conf.js b/src/sunstone/public/app/tabs/vms-tab/panels/conf.js index f852ba2e5c..25f01d144c 100644 --- a/src/sunstone/public/app/tabs/vms-tab/panels/conf.js +++ b/src/sunstone/public/app/tabs/vms-tab/panels/conf.js @@ -56,7 +56,7 @@ define(function(require) { var conf = {}; var template = this.element.TEMPLATE; - $.each(["OS", "FEATURES", "INPUT", "GRAPHICS", "RAW", "CONTEXT"], function(){ + $.each(["OS", "FEATURES", "INPUT", "GRAPHICS", "RAW", "CONTEXT", "CPU_MODEL"], function(){ if(template[this] != undefined){ conf[this] = template[this]; } diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index f149c3f97c..f93717c55c 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -2744,7 +2744,8 @@ static std::map> UPDATECONF_ATTRS = { "GUEST_AGENT"} }, {"INPUT", {"TYPE", "BUS"} }, {"GRAPHICS", {"TYPE", "LISTEN", "PASSWD", "KEYMAP"} }, - {"RAW", {"TYPE", "DATA", "DATA_VMX"} } + {"RAW", {"TYPE", "DATA", "DATA_VMX"} }, + {"CPU_MODEL", {"MODEL"} } }; /** @@ -2884,7 +2885,7 @@ int VirtualMachine::updateconf(VirtualMachineTemplate& tmpl, string &err) } // ------------------------------------------------------------------------- - // Update OS, FEATURES, INPUT, GRAPHICS, RAW + // Update OS, FEATURES, INPUT, GRAPHICS, RAW, CPU_MODEL // ------------------------------------------------------------------------- replace_vector_values(obj_template, &tmpl, "OS"); @@ -2901,6 +2902,8 @@ int VirtualMachine::updateconf(VirtualMachineTemplate& tmpl, string &err) replace_vector_values(obj_template, &tmpl, "RAW"); + replace_vector_values(obj_template, &tmpl, "CPU_MODEL"); + // ------------------------------------------------------------------------- // Update CONTEXT: any value // ------------------------------------------------------------------------- From f174b71b4ba9227589dd97a00e5d103327a1facf Mon Sep 17 00:00:00 2001 From: Daniel Clavijo Coca Date: Mon, 21 Jan 2019 09:33:40 -0600 Subject: [PATCH 10/15] F #1684: Fix wild vm nic hotplug (#2834) --- src/vmm_mad/remotes/lib/lxd/container.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vmm_mad/remotes/lib/lxd/container.rb b/src/vmm_mad/remotes/lib/lxd/container.rb index 99cf724adc..d1a70c7664 100644 --- a/src/vmm_mad/remotes/lib/lxd/container.rb +++ b/src/vmm_mad/remotes/lib/lxd/container.rb @@ -273,7 +273,7 @@ class Container # Removes the context section from the LXD configuration and unmap the # context device def detach_context - return unless @one.has_context? + return 'no context' unless @one.has_context? csrc = @lxc['devices']['context']['source'].clone From 5d9c92d0d91a1039a75edb79c722ec3a6aa91df1 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Mon, 21 Jan 2019 17:58:05 +0100 Subject: [PATCH 11/15] development: Fix pool index when the first insert fails --- src/pool/PoolSQL.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/pool/PoolSQL.cc b/src/pool/PoolSQL.cc index 0b89478a2d..392cf6e4f8 100644 --- a/src/pool/PoolSQL.cc +++ b/src/pool/PoolSQL.cc @@ -148,7 +148,14 @@ int PoolSQL::allocate(PoolObjectSQL *objsql, string& error_str) if( rc == -1 ) { - _set_lastOID(--lastOID, db, table); + lastOID = lastOID - 1; + + if ( lastOID < 0 ) + { + lastOID = 0; + } + + _set_lastOID(lastOID, db, table); } unlock(); From 4d9a1f9c72aabadccc32f132adbe873c12550414 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Mon, 21 Jan 2019 19:08:47 +0100 Subject: [PATCH 12/15] F #1684: Ad linuxcontainers market by default --- src/market/MarketPlacePool.cc | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/market/MarketPlacePool.cc b/src/market/MarketPlacePool.cc index 3fa5a86257..4f57f4ef00 100644 --- a/src/market/MarketPlacePool.cc +++ b/src/market/MarketPlacePool.cc @@ -33,12 +33,18 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave) //lastOID is set in PoolSQL::init_cb if (get_lastOID() == -1) { - // Build the default default security group + // Build the template for the OpenNebula Systems MarketPlace string default_market = "NAME=\"OpenNebula Public\"\n" "MARKET_MAD=one\n" "DESCRIPTION=\"OpenNebula Systems MarketPlace\""; + string lxc_market = + "NAME=\"Linux Containers\"\n" + "MARKET_MAD=linuxcontainers\n" + "DESCRIPTION=\"MarketPlace for the public image server fo LXC &" + " LXD hosted at linuxcontainers.org\""; + Nebula& nd = Nebula::instance(); UserPool * upool = nd.get_upool(); User * oneadmin = upool->get_ro(0); @@ -46,9 +52,12 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave) string error; MarketPlaceTemplate * default_tmpl = new MarketPlaceTemplate; + MarketPlaceTemplate * lxc_tmpl = new MarketPlaceTemplate; + char * error_parse; default_tmpl->parse(default_market, &error_parse); + lxc_tmpl->parse(lxc_market, &error_parse); MarketPlace * marketplace = new MarketPlace( oneadmin->get_uid(), @@ -58,19 +67,33 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave) oneadmin->get_umask(), default_tmpl); + MarketPlace * lxc_marketplace = new MarketPlace( + oneadmin->get_uid(), + oneadmin->get_gid(), + oneadmin->get_uname(), + oneadmin->get_gname(), + oneadmin->get_umask(), + lxc_tmpl); + oneadmin->unlock(); marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error); + lxc_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error); marketplace->zone_id = Nebula::instance().get_zone_id(); + lxc_marketplace->zone_id = Nebula::instance().get_zone_id(); marketplace->parse_template(error); + lxc_marketplace->parse_template(error); - if (PoolSQL::allocate(marketplace, error) < 0) + int rc = PoolSQL::allocate(marketplace, error); + + rc += PoolSQL::allocate(lxc_marketplace, error); + + if (rc < 0) { ostringstream oss; - oss << "Error trying to create default " - << "OpenNebula Systems MarketPlace: " << error; + oss << "Error trying to create default marketplaces: " << error; NebulaLog::log("MKP", Log::ERROR, oss); throw runtime_error(oss.str()); From 9f3b38b08683f621c5bba702ffe6e8c12499d584 Mon Sep 17 00:00:00 2001 From: sergio semedi Date: Mon, 21 Jan 2019 16:51:26 +0100 Subject: [PATCH 13/15] B #2833 [vCenter] remove duplicated unmanaged nics works --- src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb b/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb index ebdc034361..767abd3c98 100644 --- a/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb +++ b/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb @@ -1050,7 +1050,7 @@ class VirtualMachine < VCenterDriver::Template key = backing.port.portgroupKey end - @nics[key] = Nic.vc_nic(d) + @nics["#{key}#{d.key}"] = Nic.vc_nic(d) end @nics.reject{|k| k == :macs} From 5c59ff5894e375dbc977668d118810066e85ea51 Mon Sep 17 00:00:00 2001 From: sergio semedi Date: Tue, 22 Jan 2019 10:54:40 +0100 Subject: [PATCH 14/15] B #2835: [vCenter] unmanaged dportgroups are working --- .../remotes/lib/vcenter_driver/virtual_machine.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb b/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb index 767abd3c98..e212c29bf4 100644 --- a/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb +++ b/src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb @@ -1217,10 +1217,17 @@ class VirtualMachine < VCenterDriver::Template if !unmanaged_nics.empty? nics = get_vcenter_nics - select = ->(name){ + select_net =->(ref){ device = nil nics.each do |nic| - next unless nic.deviceInfo.summary == name + type = nic.backing.class + if type == NET_CARD + nref = nic.backing.network._ref + else + nref = nic.backing.port.portgroupKey + end + + next unless nref == ref device = nic break end @@ -1231,7 +1238,7 @@ class VirtualMachine < VCenterDriver::Template } unmanaged_nics.each do |unic| - vnic = select.call(unic['BRIDGE']) + vnic = select_net.call(unic['VCENTER_NET_REF']) vcenter_nic_class = vnic.class new_model = unic['MODEL'] && !unic['MODEL'].empty? && !unic['MODEL'].nil? opennebula_nic_class = nic_model_class(unic['MODEL']) if new_model From fa4d1b3a898df1a32369a892b4bc88f5193cd573 Mon Sep 17 00:00:00 2001 From: sergio semedi Date: Wed, 23 Jan 2019 14:52:46 +0100 Subject: [PATCH 15/15] B #1826: [vCenter] debug messages tm && ds --- src/datastore_mad/remotes/vcenter/clone | 4 +++- src/datastore_mad/remotes/vcenter/mkfs | 4 +++- src/datastore_mad/remotes/vcenter/monitor | 4 +++- src/datastore_mad/remotes/vcenter/rm | 4 +++- src/tm_mad/vcenter/clone | 4 +++- src/tm_mad/vcenter/cpds | 4 +++- src/tm_mad/vcenter/delete | 5 ++++- src/tm_mad/vcenter/mkimage | 4 +++- src/tm_mad/vcenter/mv | 2 ++ src/tm_mad/vcenter/mvds | 4 +++- src/tm_mad/vcenter/resize | 4 +++- 11 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/datastore_mad/remotes/vcenter/clone b/src/datastore_mad/remotes/vcenter/clone index 1fa69dfe99..762e98db82 100755 --- a/src/datastore_mad/remotes/vcenter/clone +++ b/src/datastore_mad/remotes/vcenter/clone @@ -73,7 +73,9 @@ begin rescue Exception => e message = "Error cloning img #{src_path} to #{target_ds_name}"\ " Reason: \"#{e.message}\"\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client diff --git a/src/datastore_mad/remotes/vcenter/mkfs b/src/datastore_mad/remotes/vcenter/mkfs index de07dee2b9..09e48c441f 100755 --- a/src/datastore_mad/remotes/vcenter/mkfs +++ b/src/datastore_mad/remotes/vcenter/mkfs @@ -75,7 +75,9 @@ begin rescue Exception => e message = "Error creating virtual disk #{img_name}."\ " Reason: \"#{e.message}\"\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client diff --git a/src/datastore_mad/remotes/vcenter/monitor b/src/datastore_mad/remotes/vcenter/monitor index 1dae1e5107..ad2eec3b5c 100755 --- a/src/datastore_mad/remotes/vcenter/monitor +++ b/src/datastore_mad/remotes/vcenter/monitor @@ -61,7 +61,9 @@ begin rescue Exception => e message = "Error monitoring datastore #{id}."\ " Reason: \"#{e.message}\"\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client diff --git a/src/datastore_mad/remotes/vcenter/rm b/src/datastore_mad/remotes/vcenter/rm index 72fdf0080e..8e849adb73 100755 --- a/src/datastore_mad/remotes/vcenter/rm +++ b/src/datastore_mad/remotes/vcenter/rm @@ -88,7 +88,9 @@ rescue Exception => e if !e.message.start_with?('FileNotFound') message = "Error deleting virtual disk #{img_src}."\ " Reason: \"#{e.message}\"\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 end ensure diff --git a/src/tm_mad/vcenter/clone b/src/tm_mad/vcenter/clone index c41a7ab4b9..3182493066 100755 --- a/src/tm_mad/vcenter/clone +++ b/src/tm_mad/vcenter/clone @@ -110,7 +110,9 @@ rescue Exception => e message = "Error clone virtual disk #{src_path} in "\ "datastore #{target_ds_name_vc}. "\ "Reason: #{e.message}\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client diff --git a/src/tm_mad/vcenter/cpds b/src/tm_mad/vcenter/cpds index 6463c7cd92..e29474ab14 100755 --- a/src/tm_mad/vcenter/cpds +++ b/src/tm_mad/vcenter/cpds @@ -92,7 +92,9 @@ begin rescue Exception => e message = "Error copying img #{src_path} to #{target_ds_name_vc} "\ "Reason: #{e.message}\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client diff --git a/src/tm_mad/vcenter/delete b/src/tm_mad/vcenter/delete index 38b8a42123..a0f017d440 100755 --- a/src/tm_mad/vcenter/delete +++ b/src/tm_mad/vcenter/delete @@ -104,6 +104,9 @@ begin rescue Exception => e vi_client.close_connection if vi_client - STDERR.puts "#{@error_message}. Reason: #{e.message}\n#{e.backtrace}" + message = "#{@error_message}. Reason: #{e.message}\n#{e.backtrace}" + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 end diff --git a/src/tm_mad/vcenter/mkimage b/src/tm_mad/vcenter/mkimage index eb8192dc50..a06a5246f3 100755 --- a/src/tm_mad/vcenter/mkimage +++ b/src/tm_mad/vcenter/mkimage @@ -88,7 +88,9 @@ begin rescue Exception => e message = "Error creating virtual disk in #{ds_vc['name']}."\ " Reason: #{e.message}\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client diff --git a/src/tm_mad/vcenter/mv b/src/tm_mad/vcenter/mv index 832e92200a..19b2b280e6 100755 --- a/src/tm_mad/vcenter/mv +++ b/src/tm_mad/vcenter/mv @@ -58,5 +58,7 @@ rescue StandardError => e 'failed due to '\ "\"#{e.message}\"\n" OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit(-1) end diff --git a/src/tm_mad/vcenter/mvds b/src/tm_mad/vcenter/mvds index c324c7c0f9..d4cc5b8e74 100755 --- a/src/tm_mad/vcenter/mvds +++ b/src/tm_mad/vcenter/mvds @@ -80,7 +80,9 @@ begin rescue Exception => e message = "Error detaching virtual disk #{disk_id} from vm #{vmid}."\ " Reason: #{e.message}\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client diff --git a/src/tm_mad/vcenter/resize b/src/tm_mad/vcenter/resize index 349738ab1e..3c30a2062f 100755 --- a/src/tm_mad/vcenter/resize +++ b/src/tm_mad/vcenter/resize @@ -72,7 +72,9 @@ begin rescue Exception => e message = "Error resizing disk #{disk_id} for VM #{one_vm["NAME"]} "\ "Reason: #{e.message}\n#{e.backtrace}" - STDERR.puts error_message(message) + OpenNebula.log_error(message) + STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + exit -1 ensure vi_client.close_connection if vi_client