Merge branch 'master' into feature-2816

.gitignore

@@ -28,7 +28,7 @@ src/vmm_mad/remotes/lxd/tests/
 src/oca/python/pyone/bindings
 src/oca/python/build/
 src/oca/python/dist/
-src/oca/python/opennebula.egg-info/
+src/oca/python/pyone.egg-info/
 src/oca/python/doc/
 
 src/docker_machine/pkg

install.sh

@@ -219,6 +219,7 @@ fi
 
 SHARE_DIRS="$SHARE_LOCATION/examples \
             $SHARE_LOCATION/websockify \
+            $SHARE_LOCATION/websockify/websockify \
             $SHARE_LOCATION/esx-fw-vnc \
             $SHARE_LOCATION/oneprovision"
 
@@ -477,7 +478,8 @@ INSTALL_FILES=(
     NETWORK_OVSWITCH_VXLAN_FILES:$VAR_LOCATION/remotes/vnm/ovswitch_vxlan
     NETWORK_VCENTER_FILES:$VAR_LOCATION/remotes/vnm/vcenter
     EXAMPLE_SHARE_FILES:$SHARE_LOCATION/examples
-    WEBSOCKIFY_SHARE_FILES:$SHARE_LOCATION/websockify
+    WEBSOCKIFY_SHARE_RUN_FILES:$SHARE_LOCATION/websockify
+    WEBSOCKIFY_SHARE_MODULE_FILES:$SHARE_LOCATION/websockify/websockify
     ESX_FW_VNC_SHARE_FILES:$SHARE_LOCATION/esx-fw-vnc
     INSTALL_GEMS_SHARE_FILES:$SHARE_LOCATION
     ONETOKEN_SHARE_FILE:$SHARE_LOCATION
@@ -1530,9 +1532,12 @@ EXAMPLE_SHARE_FILES="share/examples/vm.template \
 # Files required to interact with the websockify server
 #-------------------------------------------------------------------------------
 
-WEBSOCKIFY_SHARE_FILES="share/websockify/websocketproxy.py \
-                        share/websockify/websocket.py \
-                        share/websockify/websockify"
+WEBSOCKIFY_SHARE_RUN_FILES="share/websockify/run"
+WEBSOCKIFY_SHARE_MODULE_FILES="share/websockify/websockify/__init__.py \
+                               share/websockify/websockify/auth_plugins.py \
+                               share/websockify/websockify/token_plugins.py \
+                               share/websockify/websockify/websocket.py \
+                               share/websockify/websockify/websocketproxy.py"
 
 #-------------------------------------------------------------------------------
 # Installation packages for ESX hosts to enable VNC ports

share/etc/oned.conf

@@ -1307,7 +1307,7 @@ TM_MAD_CONF = [
 TM_MAD_CONF = [
     NAME = "ceph", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
     DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="mixed",
-    TM_MAD_SYSTEM = "ssh", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
+    TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
     DISK_TYPE_SSH = "FILE", TM_MAD_SYSTEM = "shared", LN_TARGET_SHARED = "NONE",
     CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "RBD"
 ]

share/websockify/run

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+import websockify
+
+websockify.websocketproxy.websockify_init()

share/websockify/websockify

@@ -1 +0,0 @@
-websocketproxy.py

share/websockify/websockify/__init__.py

@@ -0,0 +1,2 @@
+from websockify.websocket import *
+from websockify.websocketproxy import *

share/websockify/websockify/auth_plugins.py

@@ -0,0 +1,83 @@
+class BasePlugin(object):
+    def __init__(self, src=None):
+        self.source = src
+
+    def authenticate(self, headers, target_host, target_port):
+        pass
+
+
+class AuthenticationError(Exception):
+    def __init__(self, log_msg=None, response_code=403, response_headers={}, response_msg=None):
+        self.code = response_code
+        self.headers = response_headers
+        self.msg = response_msg
+
+        if log_msg is None:
+            log_msg = response_msg
+
+        super(AuthenticationError, self).__init__('%s %s' % (self.code, log_msg))
+
+
+class InvalidOriginError(AuthenticationError):
+    def __init__(self, expected, actual):
+        self.expected_origin = expected
+        self.actual_origin = actual
+
+        super(InvalidOriginError, self).__init__(
+            response_msg='Invalid Origin',
+            log_msg="Invalid Origin Header: Expected one of "
+                    "%s, got '%s'" % (expected, actual))
+
+
+class BasicHTTPAuth(object):
+    """Verifies Basic Auth headers. Specify src as username:password"""
+
+    def __init__(self, src=None):
+        self.src = src
+
+    def authenticate(self, headers, target_host, target_port):
+        import base64
+        auth_header = headers.get('Authorization')
+        if auth_header:
+            if not auth_header.startswith('Basic '):
+                raise AuthenticationError(response_code=403)
+
+            try:
+                user_pass_raw = base64.b64decode(auth_header[6:])
+            except TypeError:
+                raise AuthenticationError(response_code=403)
+
+            try:
+                # http://stackoverflow.com/questions/7242316/what-encoding-should-i-use-for-http-basic-authentication
+                user_pass_as_text = user_pass_raw.decode('ISO-8859-1')
+            except UnicodeDecodeError:
+                raise AuthenticationError(response_code=403)
+
+            user_pass = user_pass_as_text.split(':', 1)
+            if len(user_pass) != 2:
+                raise AuthenticationError(response_code=403)
+
+            if not self.validate_creds(*user_pass):
+                raise AuthenticationError(response_code=403)
+
+        else:
+            raise AuthenticationError(response_code=401,
+                                      response_headers={'WWW-Authenticate': 'Basic realm="Websockify"'})
+
+    def validate_creds(self, username, password):
+        if '%s:%s' % (username, password) == self.src:
+            return True
+        else:
+            return False
+
+class ExpectOrigin(object):
+    def __init__(self, src=None):
+        if src is None:
+            self.source = []
+        else:
+            self.source = src.split()
+
+    def authenticate(self, headers, target_host, target_port):
+        origin = headers.get('Origin', None)
+        if origin is None or origin not in self.source:
+            raise InvalidOriginError(expected=self.source, actual=origin)

share/websockify/websockify/token_plugins.py

@@ -0,0 +1,83 @@
+import os
+
+class BasePlugin(object):
+    def __init__(self, src):
+        self.source = src
+
+    def lookup(self, token):
+        return None
+
+
+class ReadOnlyTokenFile(BasePlugin):
+    # source is a token file with lines like
+    #   token: host:port
+    # or a directory of such files
+    def __init__(self, *args, **kwargs):
+        super(ReadOnlyTokenFile, self).__init__(*args, **kwargs)
+        self._targets = None
+
+    def _load_targets(self):
+        if os.path.isdir(self.source):
+            cfg_files = [os.path.join(self.source, f) for
+                         f in os.listdir(self.source)]
+        else:
+            cfg_files = [self.source]
+
+        self._targets = {}
+        for f in cfg_files:
+            for line in [l.strip() for l in open(f).readlines()]:
+                if line and not line.startswith('#'):
+                    tok, target = line.split(': ')
+                    self._targets[tok] = target.strip().rsplit(':', 1)
+
+    def lookup(self, token):
+        if self._targets is None:
+            self._load_targets()
+
+        if token in self._targets:
+            return self._targets[token]
+        else:
+            return None
+
+
+# the above one is probably more efficient, but this one is
+# more backwards compatible (although in most cases
+# ReadOnlyTokenFile should suffice)
+class TokenFile(ReadOnlyTokenFile):
+    # source is a token file with lines like
+    #   token: host:port
+    # or a directory of such files
+    def lookup(self, token):
+        self._load_targets()
+
+        return super(TokenFile, self).lookup(token)
+
+
+class BaseTokenAPI(BasePlugin):
+    # source is a url with a '%s' in it where the token
+    # should go
+
+    # we import things on demand so that other plugins
+    # in this file can be used w/o unecessary dependencies
+
+    def process_result(self, resp):
+        return resp.text.split(':')
+
+    def lookup(self, token):
+        import requests
+
+        resp = requests.get(self.source % token)
+
+        if resp.ok:
+            return self.process_result(resp)
+        else:
+            return None
+
+
+class JSONTokenApi(BaseTokenAPI):
+    # source is a url with a '%s' in it where the token
+    # should go
+
+    def process_result(self, resp):
+        resp_json = resp.json()
+        return (resp_json['host'], resp_json['port'])

share/websockify/websockify/websocket.py

@@ -104,6 +104,8 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
         self.handler_id = getattr(server, "handler_id", False)
         self.file_only = getattr(server, "file_only", False)
         self.traffic = getattr(server, "traffic", False)
+        self.auto_pong = getattr(server, "auto_pong", False)
+        self.strict_mode = getattr(server, "strict_mode", True)
 
         self.logger = getattr(server, "logger", None)
         if self.logger is None:
@@ -111,6 +113,9 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
 
         SimpleHTTPRequestHandler.__init__(self, req, addr, server)
 
+    def log_message(self, format, *args):
+        self.logger.info("%s - - [%s] %s" % (self.address_string(), self.log_date_time_string(), format % args))
+
     @staticmethod
     def unmask(buf, hlen, plen):
         pstart = hlen + 4
@@ -118,20 +123,24 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
         if numpy:
             b = c = s2b('')
             if plen >= 4:
-                mask = numpy.frombuffer(buf, dtype=numpy.dtype('<u4'),
-                        offset=hlen, count=1)
-                data = numpy.frombuffer(buf, dtype=numpy.dtype('<u4'),
-                        offset=pstart, count=int(plen / 4))
+                dtype=numpy.dtype('<u4')
+                if sys.byteorder == 'big':
+                    dtype = dtype.newbyteorder('>')
+                mask = numpy.frombuffer(buf, dtype, offset=hlen, count=1)
+                data = numpy.frombuffer(buf, dtype, offset=pstart,
+                        count=int(plen / 4))
                 #b = numpy.bitwise_xor(data, mask).data
                 b = numpy.bitwise_xor(data, mask).tostring()
 
             if plen % 4:
                 #self.msg("Partial unmask")
-                mask = numpy.frombuffer(buf, dtype=numpy.dtype('B'),
-                        offset=hlen, count=(plen % 4))
-                data = numpy.frombuffer(buf, dtype=numpy.dtype('B'),
-                        offset=pend - (plen % 4),
+                dtype=numpy.dtype('B')
+                if sys.byteorder == 'big':
+                    dtype = dtype.newbyteorder('>')
+                mask = numpy.frombuffer(buf, dtype, offset=hlen,
                         count=(plen % 4))
+                data = numpy.frombuffer(buf, dtype,
+                        offset=pend - (plen % 4), count=(plen % 4))
                 c = numpy.bitwise_xor(data, mask).tostring()
             return b + c
         else:
@@ -172,7 +181,7 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
         return header + buf, len(header), 0
 
     @staticmethod
-    def decode_hybi(buf, base64=False, logger=None):
+    def decode_hybi(buf, base64=False, logger=None, strict=True):
         """ Decode HyBi style WebSocket packets.
         Returns:
             {'fin'          : 0_or_1,
@@ -238,6 +247,10 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
                                                   f['length'])
         else:
             logger.debug("Unmasked frame: %s" % repr(buf))
+
+            if strict:
+                raise WebSocketRequestHandler.CClose(1002, "The client sent an unmasked frame.")
+
             f['payload'] = buf[(f['hlen'] + f['masked'] * 4):full_len]
 
         if base64 and f['opcode'] in [1, 2]:
@@ -346,7 +359,8 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
 
         while buf:
             frame = self.decode_hybi(buf, base64=self.base64,
-                                     logger=self.logger)
+                                     logger=self.logger,
+                                     strict=self.strict_mode)
             #self.msg("Received buf: %s, frame: %s", repr(buf), frame)
 
             if frame['payload'] == None:
@@ -360,6 +374,15 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
                     closed = {'code': frame['close_code'],
                               'reason': frame['close_reason']}
                     break
+                elif self.auto_pong and frame['opcode'] == 0x9: # ping
+                    self.print_traffic("} ping %s\n" %
+                        repr(frame['payload']))
+                    self.send_pong(frame['payload'])
+                    return [], False
+                elif frame['opcode'] == 0xA: # pong
+                    self.print_traffic("} pong %s\n" %
+                        repr(frame['payload']))
+                    return [], False
 
             self.print_traffic("}")
 
@@ -388,10 +411,20 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
     def send_close(self, code=1000, reason=''):
         """ Send a WebSocket orderly close frame. """
 
-        msg = pack(">H%ds" % len(reason), code, reason)
+        msg = pack(">H%ds" % len(reason), code, s2b(reason))
         buf, h, t = self.encode_hybi(msg, opcode=0x08, base64=False)
         self.request.send(buf)
 
+    def send_pong(self, data=''):
+        """ Send a WebSocket pong frame. """
+        buf, h, t = self.encode_hybi(s2b(data), opcode=0x0A, base64=False)
+        self.request.send(buf)
+
+    def send_ping(self, data=''):
+        """ Send a WebSocket ping frame. """
+        buf, h, t = self.encode_hybi(s2b(data), opcode=0x09, base64=False)
+        self.request.send(buf)
+
     def do_websocket_handshake(self):
         h = self.headers
 
@@ -444,9 +477,13 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
         """Upgrade a connection to Websocket, if requested. If this succeeds,
         new_websocket_client() will be called. Otherwise, False is returned.
         """
+
         if (self.headers.get('upgrade') and
             self.headers.get('upgrade').lower() == 'websocket'):
 
+            # ensure connection is authorized, and determine the target
+            self.validate_connection()
+
             if not self.do_websocket_handshake():
                 return False
 
@@ -519,6 +556,10 @@ class WebSocketRequestHandler(SimpleHTTPRequestHandler):
         """ Do something with a WebSockets client connection. """
         raise Exception("WebSocketRequestHandler.new_websocket_client() must be overloaded")
 
+    def validate_connection(self):
+        """ Ensure that the connection is a valid connection, and set the target. """
+        pass
+
     def do_HEAD(self):
         if self.only_upgrade:
             self.send_error(405, "Method Not Allowed")
@@ -567,7 +608,7 @@ class WebSocketServer(object):
             file_only=False,
             run_once=False, timeout=0, idle_timeout=0, traffic=False,
             tcp_keepalive=True, tcp_keepcnt=None, tcp_keepidle=None,
-            tcp_keepintvl=None):
+            tcp_keepintvl=None, auto_pong=False, strict_mode=True):
 
         # settings
         self.RequestHandlerClass = RequestHandlerClass
@@ -581,6 +622,8 @@ class WebSocketServer(object):
         self.timeout        = timeout
         self.idle_timeout   = idle_timeout
         self.traffic        = traffic
+        self.file_only      = file_only
+        self.strict_mode    = strict_mode
 
         self.launch_time    = time.time()
         self.ws_connection  = False
@@ -592,6 +635,7 @@ class WebSocketServer(object):
         self.tcp_keepidle   = tcp_keepidle
         self.tcp_keepintvl  = tcp_keepintvl
 
+        self.auto_pong      = auto_pong
         # Make paths settings absolute
         self.cert = os.path.abspath(cert)
         self.key = self.web = self.record = ''
@@ -618,7 +662,10 @@ class WebSocketServer(object):
                 self.listen_host, self.listen_port)
         self.msg("  - Flash security policy server")
         if self.web:
-            self.msg("  - Web server. Web root: %s", self.web)
+            if self.file_only:
+                self.msg("  - Web server (no directory listings). Web root: %s", self.web)
+            else:
+                self.msg("  - Web server. Web root: %s", self.web)
         if ssl:
             if os.path.exists(self.cert):
                 self.msg("  - SSL/TLS support")
@@ -701,6 +748,10 @@ class WebSocketServer(object):
 
     @staticmethod
     def daemonize(keepfd=None, chdir='/'):
+
+        if keepfd is None:
+            keepfd = []
+
         os.umask(0)
         if chdir:
             os.chdir(chdir)
@@ -723,7 +774,7 @@ class WebSocketServer(object):
         if maxfd == resource.RLIM_INFINITY: maxfd = 256
         for fd in reversed(range(maxfd)):
             try:
-                if fd != keepfd:
+                if fd not in keepfd:
                     os.close(fd)
             except OSError:
                 _, exc, _ = sys.exc_info()
@@ -753,7 +804,7 @@ class WebSocketServer(object):
         """
         ready = select.select([sock], [], [], 3)[0]
 
-        
+
         if not ready:
             raise self.EClose("ignoring socket not ready")
         # Peek, but do not read the data so that we have a opportunity
@@ -761,7 +812,7 @@ class WebSocketServer(object):
         handshake = sock.recv(1024, socket.MSG_PEEK)
         #self.msg("Handshake [%s]" % handshake)
 
-        if handshake == "":
+        if not handshake:
             raise self.EClose("ignoring empty handshake")
 
         elif handshake.startswith(s2b("<policy-file-request/>")):
@@ -844,11 +895,14 @@ class WebSocketServer(object):
         raise self.Terminate()
 
     def multiprocessing_SIGCHLD(self, sig, stack):
-        self.vmsg('Reaping zombies, active child count is %s', len(multiprocessing.active_children()))
+        # TODO: figure out a way to actually log this information without
+        #       calling `log` in the signal handlers
+        multiprocessing.active_children()
 
     def fallback_SIGCHLD(self, sig, stack):
         # Reap zombies when using os.fork() (python 2.4)
-        self.vmsg("Got SIGCHLD, reaping zombies")
+        # TODO: figure out a way to actually log this information without
+        #       calling `log` in the signal handlers
         try:
             result = os.waitpid(-1, os.WNOHANG)
             while result[0]:
@@ -858,16 +912,18 @@ class WebSocketServer(object):
             pass
 
     def do_SIGINT(self, sig, stack):
-        self.msg("Got SIGINT, exiting")
+        # TODO: figure out a way to actually log this information without
+        #       calling `log` in the signal handlers
         self.terminate()
 
     def do_SIGTERM(self, sig, stack):
-        self.msg("Got SIGTERM, exiting")
+        # TODO: figure out a way to actually log this information without
+        #       calling `log` in the signal handlers
         self.terminate()
 
     def top_new_client(self, startsock, address):
         """ Do something with a WebSockets client connection. """
-        # handler process        
+        # handler process
         client = None
         try:
             try:
@@ -890,6 +946,18 @@ class WebSocketServer(object):
                 # Original socket closed by caller
                 client.close()
 
+    def get_log_fd(self):
+        """
+        Get file descriptors for the loggers.
+        They should not be closed when the process is forked.
+        """
+        descriptors = []
+        for handler in self.logger.parent.handlers:
+            if isinstance(handler, logging.FileHandler):
+                descriptors.append(handler.stream.fileno())
+
+        return descriptors
+
     def start_server(self):
         """
         Daemonize if requested. Listen for for connections. Run
@@ -905,7 +973,9 @@ class WebSocketServer(object):
                             tcp_keepintvl=self.tcp_keepintvl)
 
         if self.daemon:
-            self.daemonize(keepfd=lsock.fileno(), chdir=self.web)
+            keepfd = self.get_log_fd()
+            keepfd.append(lsock.fileno())
+            self.daemonize(keepfd=keepfd, chdir=self.web)
 
         self.started()  # Some things need to happen after daemonizing
 
@@ -1009,8 +1079,17 @@ class WebSocketServer(object):
 
                     except (self.Terminate, SystemExit, KeyboardInterrupt):
                         self.msg("In exit")
+                        # terminate all child processes
+                        if multiprocessing and not self.run_once:
+                            children = multiprocessing.active_children()
+
+                            for child in children:
+                                self.msg("Terminating child %s" % child.pid)
+                                child.terminate()
+
                         break
                     except Exception:
+                        exc = sys.exc_info()[1]
                         self.msg("handler exception: %s", str(exc))
                         self.vmsg("exception", exc_info=True)
 

share/websockify/websockify/websocketproxy.py

@@ -11,13 +11,14 @@ as taken from http://docs.python.org/dev/library/ssl.html#certificates
 
 '''
 
-import signal, socket, optparse, time, os, sys, subprocess, logging
+import signal, socket, optparse, time, os, sys, subprocess, logging, errno
 try:    from socketserver import ForkingMixIn
 except: from SocketServer import ForkingMixIn
 try:    from http.server import HTTPServer
 except: from BaseHTTPServer import HTTPServer
-from select import select
-import websocket
+import select
+from websockify import websocket
+from websockify import auth_plugins as auth
 try:
     from urllib.parse import parse_qs, urlparse
 except:
@@ -38,14 +39,33 @@ Traffic Legend:
     <. - Client send partial
 """
 
+    def send_auth_error(self, ex):
+        self.send_response(ex.code, ex.msg)
+        self.send_header('Content-Type', 'text/html')
+        for name, val in ex.headers.items():
+            self.send_header(name, val)
+
+        self.end_headers()
+
+    def validate_connection(self):
+        if self.server.token_plugin:
+            (self.server.target_host, self.server.target_port) = self.get_target(self.server.token_plugin, self.path)
+
+        if self.server.auth_plugin:
+            try:
+                self.server.auth_plugin.authenticate(
+                    headers=self.headers, target_host=self.server.target_host,
+                    target_port=self.server.target_port)
+            except auth.AuthenticationError:
+                ex = sys.exc_info()[1]
+                self.send_auth_error(ex)
+                raise
+
     def new_websocket_client(self):
         """
         Called after a new WebSocket connection has been established.
         """
-        # Checks if we receive a token, and look
-        # for a valid target for it then
-        if self.server.target_cfg:
-            (self.server.target_host, self.server.target_port) = self.get_target(self.server.target_cfg, self.path)
+        # Checking for a token is done in validate_connection()
 
         # Connect to the target
         if self.server.wrap_cmd:
@@ -73,15 +93,15 @@ Traffic Legend:
             if tsock:
                 tsock.shutdown(socket.SHUT_RDWR)
                 tsock.close()
-                if self.verbose: 
+                if self.verbose:
                     self.log_message("%s:%s: Closed target",
                             self.server.target_host, self.server.target_port)
             raise
 
-    def get_target(self, target_cfg, path):
+    def get_target(self, target_plugin, path):
         """
-        Parses the path, extracts a token, and looks for a valid
-        target for that token in the configuration file(s). Sets
+        Parses the path, extracts a token, and looks up a target
+        for that token using the token plugin. Sets
         target_host and target_port if successful
         """
         # The files in targets contain the lines
@@ -90,32 +110,17 @@ Traffic Legend:
         # Extract the token parameter from url
         args = parse_qs(urlparse(path)[4]) # 4 is the query from url
 
-        if not args.has_key('token') or not len(args['token']):
-            raise self.EClose("Token not present")
+        if not 'token' in args or not len(args['token']):
+            raise self.server.EClose("Token not present")
 
         token = args['token'][0].rstrip('\n')
 
-        # target_cfg can be a single config file or directory of
-        # config files
-        if os.path.isdir(target_cfg):
-            cfg_files = [os.path.join(target_cfg, f)
-                         for f in os.listdir(target_cfg)]
+        result_pair = target_plugin.lookup(token)
+
+        if result_pair is not None:
+            return result_pair
         else:
-            cfg_files = [target_cfg]
-
-        targets = {}
-        for f in cfg_files:
-            for line in [l.strip() for l in file(f).readlines()]:
-                if line and not line.startswith('#'):
-                    ttoken, target = line.split(': ')
-                    targets[ttoken] = target.strip()
-
-        self.vmsg("Target config: %s" % repr(targets))
-
-        if targets.has_key(token):
-            return targets[token].split(':')
-        else:
-            raise self.EClose("Token '%s' not found" % token)
+            raise self.server.EClose("Token '%s' not found" % token)
 
     def do_proxy(self, target):
         """
@@ -126,12 +131,37 @@ Traffic Legend:
         tqueue = []
         rlist = [self.request, target]
 
+        if self.server.heartbeat:
+            now = time.time()
+            self.heartbeat = now + self.server.heartbeat
+        else:
+            self.heartbeat = None
+
         while True:
             wlist = []
 
+            if self.heartbeat is not None:
+                now = time.time()
+                if now > self.heartbeat:
+                    self.heartbeat = now + self.server.heartbeat
+                    self.send_ping()
+
             if tqueue: wlist.append(target)
             if cqueue or c_pend: wlist.append(self.request)
-            ins, outs, excepts = select(rlist, wlist, [], 1)
+            try:
+                ins, outs, excepts = select.select(rlist, wlist, [], 1)
+            except (select.error, OSError):
+                exc = sys.exc_info()[1]
+                if hasattr(exc, 'errno'):
+                    err = exc.errno
+                else:
+                    err = exc[0]
+
+                if err != errno.EINTR:
+                    raise
+                else:
+                    continue
+
             if excepts: raise Exception("Socket exception")
 
             if self.request in outs:
@@ -147,7 +177,7 @@ Traffic Legend:
 
                 if closed:
                     # TODO: What about blocking on client socket?
-                    if self.verbose: 
+                    if self.verbose:
                         self.log_message("%s:%s: Client closed connection",
                                 self.server.target_host, self.server.target_port)
                     raise self.CClose(closed['code'], closed['reason'])
@@ -195,7 +225,11 @@ class WebSocketProxy(websocket.WebSocketServer):
         self.wrap_mode      = kwargs.pop('wrap_mode', None)
         self.unix_target    = kwargs.pop('unix_target', None)
         self.ssl_target     = kwargs.pop('ssl_target', None)
-        self.target_cfg     = kwargs.pop('target_cfg', None)
+        self.heartbeat      = kwargs.pop('heartbeat', None)
+
+        self.token_plugin = kwargs.pop('token_plugin', None)
+        self.auth_plugin = kwargs.pop('auth_plugin', None)
+
         # Last 3 timestamps command was run
         self.wrap_times    = [0, 0, 0]
 
@@ -251,9 +285,9 @@ class WebSocketProxy(websocket.WebSocketServer):
         else:
             dst_string = "%s:%s" % (self.target_host, self.target_port)
 
-        if self.target_cfg:
-            msg = "  - proxying from %s:%s to targets in %s" % (
-                self.listen_host, self.listen_port, self.target_cfg)
+        if self.token_plugin:
+            msg = "  - proxying from %s:%s to targets generated by %s" % (
+                self.listen_host, self.listen_port, type(self.token_plugin).__name__)
         else:
             msg = "  - proxying from %s:%s to %s" % (
                 self.listen_host, self.listen_port, dst_string)
@@ -352,20 +386,69 @@ def websockify_init():
     parser.add_option("--prefer-ipv6", "-6",
             action="store_true", dest="source_is_ipv6",
             help="prefer IPv6 when resolving source_addr")
+    parser.add_option("--libserver", action="store_true",
+            help="use Python library SocketServer engine")
     parser.add_option("--target-config", metavar="FILE",
             dest="target_cfg",
             help="Configuration file containing valid targets "
             "in the form 'token: host:port' or, alternatively, a "
-            "directory containing configuration files of this form")
-    parser.add_option("--libserver", action="store_true",
-            help="use Python library SocketServer engine")
+            "directory containing configuration files of this form "
+            "(DEPRECATED: use `--token-plugin TokenFile --token-source "
+            " path/to/token/file` instead)")
+    parser.add_option("--token-plugin", default=None, metavar="PLUGIN",
+                      help="use the given Python class to process tokens "
+                           "into host:port pairs")
+    parser.add_option("--token-source", default=None, metavar="ARG",
+                      help="an argument to be passed to the token plugin"
+                           "on instantiation")
+    parser.add_option("--auth-plugin", default=None, metavar="PLUGIN",
+                      help="use the given Python class to determine if "
+                           "a connection is allowed")
+    parser.add_option("--auth-source", default=None, metavar="ARG",
+                      help="an argument to be passed to the auth plugin"
+                           "on instantiation")
+    parser.add_option("--auto-pong", action="store_true",
+            help="Automatically respond to ping frames with a pong")
+    parser.add_option("--heartbeat", type=int, default=0,
+            help="send a ping to the client every HEARTBEAT seconds")
+    parser.add_option("--log-file", metavar="FILE",
+            dest="log_file",
+            help="File where logs will be saved")
+
+
     (opts, args) = parser.parse_args()
 
+    if opts.log_file:
+        opts.log_file = os.path.abspath(opts.log_file)
+        handler = logging.FileHandler(opts.log_file)
+        handler.setLevel(logging.DEBUG)
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        logging.getLogger(WebSocketProxy.log_prefix).addHandler(handler)
+
+    del opts.log_file
+
     if opts.verbose:
         logging.getLogger(WebSocketProxy.log_prefix).setLevel(logging.DEBUG)
 
+    if opts.token_source and not opts.token_plugin:
+        parser.error("You must use --token-plugin to use --token-source")
+
+    if opts.auth_source and not opts.auth_plugin:
+        parser.error("You must use --auth-plugin to use --auth-source")
+
+
+    # Transform to absolute path as daemon may chdir
+    if opts.target_cfg:
+        opts.target_cfg = os.path.abspath(opts.target_cfg)
+
+    if opts.target_cfg:
+        opts.token_plugin = 'TokenFile'
+        opts.token_source = opts.target_cfg
+
+    del opts.target_cfg
+
     # Sanity checks
-    if len(args) < 2 and not (opts.target_cfg or opts.unix_target):
+    if len(args) < 2 and not (opts.token_plugin or opts.unix_target):
         parser.error("Too few arguments")
     if sys.argv.count('--'):
         opts.wrap_cmd = args[1:]
@@ -390,7 +473,7 @@ def websockify_init():
     try:    opts.listen_port = int(opts.listen_port)
     except: parser.error("Error parsing listen port")
 
-    if opts.wrap_cmd or opts.unix_target or opts.target_cfg:
+    if opts.wrap_cmd or opts.unix_target or opts.token_plugin:
         opts.target_host = None
         opts.target_port = None
     else:
@@ -402,9 +485,32 @@ def websockify_init():
         try:    opts.target_port = int(opts.target_port)
         except: parser.error("Error parsing target port")
 
-    # Transform to absolute path as daemon may chdir
-    if opts.target_cfg:
-        opts.target_cfg = os.path.abspath(opts.target_cfg)
+    if opts.token_plugin is not None:
+        if '.' not in opts.token_plugin:
+            opts.token_plugin = (
+                'websockify.token_plugins.%s' % opts.token_plugin)
+
+        token_plugin_module, token_plugin_cls = opts.token_plugin.rsplit('.', 1)
+
+        __import__(token_plugin_module)
+        token_plugin_cls = getattr(sys.modules[token_plugin_module], token_plugin_cls)
+
+        opts.token_plugin = token_plugin_cls(opts.token_source)
+
+    del opts.token_source
+
+    if opts.auth_plugin is not None:
+        if '.' not in opts.auth_plugin:
+            opts.auth_plugin = 'websockify.auth_plugins.%s' % opts.auth_plugin
+
+        auth_plugin_module, auth_plugin_cls = opts.auth_plugin.rsplit('.', 1)
+
+        __import__(auth_plugin_module)
+        auth_plugin_cls = getattr(sys.modules[auth_plugin_module], auth_plugin_cls)
+
+        opts.auth_plugin = auth_plugin_cls(opts.auth_source)
+
+    del opts.auth_source
 
     # Create and start the WebSockets proxy
     libserver = opts.libserver
@@ -433,9 +539,13 @@ class LibProxyServer(ForkingMixIn, HTTPServer):
         self.wrap_mode      = kwargs.pop('wrap_mode', None)
         self.unix_target    = kwargs.pop('unix_target', None)
         self.ssl_target     = kwargs.pop('ssl_target', None)
-        self.target_cfg     = kwargs.pop('target_cfg', None)
+        self.token_plugin   = kwargs.pop('token_plugin', None)
+        self.auth_plugin    = kwargs.pop('auth_plugin', None)
+        self.heartbeat      = kwargs.pop('heartbeat', None)
+
+        self.token_plugin = None
+        self.auth_plugin = None
         self.daemon = False
-        self.target_cfg = None
 
         # Server configuration
         listen_host    = kwargs.pop('listen_host', '')
@@ -456,8 +566,8 @@ class LibProxyServer(ForkingMixIn, HTTPServer):
 
         if web:
             os.chdir(web)
-            
-        HTTPServer.__init__(self, (listen_host, listen_port), 
+
+        HTTPServer.__init__(self, (listen_host, listen_port),
                             RequestHandlerClass)
 
 

src/cli/onevm

@@ -1131,11 +1131,12 @@ CommandParser::CmdParser.new(ARGV) do
         This command accepts a template file or opens an editor, the full list
         of configuration attributes are:
 
-        OS       = ["ARCH", "MACHINE", "KERNEL", "INITRD", "BOOTLOADER", "BOOT"]
-        FEATURES = ["ACPI", "PAE", "APIC", "LOCALTIME", "HYPERV", "GUEST_AGENT"]
-        INPUT    = ["TYPE", "BUS"]
-        GRAPHICS = ["TYPE", "LISTEN", "PASSWD", "KEYMAP" ]
-        RAW      = ["DATA", "DATA_VMX", "TYPE"]
+        OS        = ["ARCH", "MACHINE", "KERNEL", "INITRD", "BOOTLOADER", "BOOT"]
+        FEATURES  = ["ACPI", "PAE", "APIC", "LOCALTIME", "HYPERV", "GUEST_AGENT"]
+        INPUT     = ["TYPE", "BUS"]
+        GRAPHICS  = ["TYPE", "LISTEN", "PASSWD", "KEYMAP" ]
+        RAW       = ["DATA", "DATA_VMX", "TYPE"]
+        CPU_MODEL = ["MODEL"]
         CONTEXT (any value, **variable substitution will be made**)
     EOT
 
@@ -1156,7 +1157,7 @@ CommandParser::CmdParser.new(ARGV) do
 
                 template = vm.template_like_str('TEMPLATE', true,
                                                 'OS | FEATURES | INPUT | '\
-                                                'GRAPHICS | RAW | CONTEXT')
+                                                'GRAPHICS | RAW | CONTEXT | CPU_MODEL')
                 template = OpenNebulaHelper.editor_input(template)
             end
 

src/datastore_mad/remotes/libfs.sh

@@ -450,6 +450,33 @@ function check_restricted {
     echo 0
 }
 
+#-------------------------------------------------------------------------------
+# Filter out hosts which are OFF, ERROR or DISABLED
+#   @param $1 - space separated list of hosts
+#   @return   - space separated list of hosts which are not in OFF, ERROR or
+#               DISABLED sate
+#-------------------------------------------------------------------------------
+function remove_off_hosts {
+    ALL_HOSTS_ARRAY=($1)
+    OFF_HOSTS_STR=$(onehost list --no-pager --csv \
+		--filter="STAT=off,STAT=err,STAT=dsbl" --list=NAME,STAT 2>/dev/null)
+
+    if [ $? -eq 0 ]; then
+        OFF_HOSTS_ARRAY=($( echo "$OFF_HOSTS_STR" | awk -F, '{ if (NR>1) print $1 }'))
+        for HOST in "${ALL_HOSTS_ARRAY[@]}"; do
+            OFF=false
+            for OFF_HOST in "${OFF_HOSTS_ARRAY[@]}"; do
+                [ $HOST = $OFF_HOST ] && { OFF=true; break; }
+            done
+            $OFF || echo -ne "$HOST "
+        done
+    else
+        # onehost cmd failed, can't filter anything, better return unchanged
+        echo $1
+        return 1
+    fi
+}
+
 #-------------------------------------------------------------------------------
 # Gets the host to be used as bridge to talk to the storage system
 # Implements a round robin for the bridges
@@ -458,7 +485,14 @@ function check_restricted {
 #   @return host to be used as bridge
 #-------------------------------------------------------------------------------
 function get_destination_host {
-    HOSTS_ARRAY=($BRIDGE_LIST)
+    REDUCED_LIST=$(remove_off_hosts "$BRIDGE_LIST")
+
+    if [ -z "$REDUCED_LIST" -a -n "$BRIDGE_LIST" ]; then
+        error_message "All hosts from 'BRIDGE_LIST' are offline, error or disabled"
+        exit -1
+    fi
+
+    HOSTS_ARRAY=($REDUCED_LIST)
     N_HOSTS=${#HOSTS_ARRAY[@]}
 
     if [ -n "$1" ]; then

src/datastore_mad/remotes/vcenter/clone

@@ -73,7 +73,9 @@ begin
 rescue Exception => e
     message = "Error cloning img #{src_path} to #{target_ds_name}"\
               " Reason: \"#{e.message}\"\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/datastore_mad/remotes/vcenter/cp

@@ -57,15 +57,9 @@ md5          = drv_action["/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5"]
 sha1         = drv_action["/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1"]
 nodecomp     = drv_action["/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS"]
 limit_bw     = drv_action["/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW"]
-format       = drv_action["IMAGE/TEMPLATE/FORMAT"]
 
 check_valid img_path, "img_path"
 
-if format != 'vmdk' && format != 'iso'
-    one_img = VCenterDriver::VIHelper.one_item(OpenNebula::Image, id)
-    one_img.replace({'FORMAT' => 'vmdk'})
-end
-
 # if image is already in a vCenter datastore return the path
 if img_path.start_with? "vcenter://"
     img_path = img_path.sub("vcenter://", "")

src/datastore_mad/remotes/vcenter/mkfs

@@ -75,7 +75,9 @@ begin
 rescue Exception => e
     message = "Error creating virtual disk #{img_name}."\
               " Reason: \"#{e.message}\"\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/datastore_mad/remotes/vcenter/monitor

@@ -61,7 +61,9 @@ begin
 rescue Exception => e
     message = "Error monitoring datastore #{id}."\
               " Reason: \"#{e.message}\"\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/datastore_mad/remotes/vcenter/rm

@@ -88,7 +88,9 @@ rescue Exception => e
     if !e.message.start_with?('FileNotFound')
         message = "Error deleting virtual disk #{img_src}."\
                   " Reason: \"#{e.message}\"\n#{e.backtrace}"
-        STDERR.puts error_message(message)
+        OpenNebula.log_error(message)
+        STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
         exit -1
     end
 ensure

src/mad/ruby/scripts_common.rb

@@ -75,11 +75,11 @@ module OpenNebula
     # Executes a command, if it fails returns error message and exits
     # If a second parameter is present it is used as the error message when
     # the command fails
-    def self.exec_and_log(command, message=nil)
+    def self.exec_and_log(command, message=nil, allowed_return_code=0)
         output=`#{command} 2>&1 1>/dev/null`
         code=$?.exitstatus
 
-        if code!=0
+        if code!=0 && code!=allowed_return_code
             log_error "Command \"#{command}\" failed."
             log_error output
             if !message

src/market/MarketPlacePool.cc

@@ -33,12 +33,18 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
     //lastOID is set in PoolSQL::init_cb
     if (get_lastOID() == -1)
     {
-        // Build the default default security group
+        // Build the template for the OpenNebula Systems MarketPlace
         string default_market =
             "NAME=\"OpenNebula Public\"\n"
             "MARKET_MAD=one\n"
             "DESCRIPTION=\"OpenNebula Systems MarketPlace\"";
 
+        string lxc_market =
+            "NAME=\"Linux Containers\"\n"
+            "MARKET_MAD=linuxcontainers\n"
+            "DESCRIPTION=\"MarketPlace for the public image server fo LXC &"
+            " LXD hosted at linuxcontainers.org\"";
+
         Nebula& nd         = Nebula::instance();
         UserPool * upool   = nd.get_upool();
         User *    oneadmin = upool->get_ro(0);
@@ -46,9 +52,12 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
         string error;
 
         MarketPlaceTemplate * default_tmpl = new MarketPlaceTemplate;
+        MarketPlaceTemplate * lxc_tmpl     = new MarketPlaceTemplate;
+
         char * error_parse;
 
         default_tmpl->parse(default_market, &error_parse);
+        lxc_tmpl->parse(lxc_market, &error_parse);
 
         MarketPlace * marketplace = new MarketPlace(
                 oneadmin->get_uid(),
@@ -58,19 +67,33 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
                 oneadmin->get_umask(),
                 default_tmpl);
 
+        MarketPlace * lxc_marketplace = new MarketPlace(
+                oneadmin->get_uid(),
+                oneadmin->get_gid(),
+                oneadmin->get_uname(),
+                oneadmin->get_gname(),
+                oneadmin->get_umask(),
+                lxc_tmpl);
+
         oneadmin->unlock();
 
         marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
+        lxc_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
 
         marketplace->zone_id = Nebula::instance().get_zone_id();
+        lxc_marketplace->zone_id = Nebula::instance().get_zone_id();
 
         marketplace->parse_template(error);
+        lxc_marketplace->parse_template(error);
 
-        if (PoolSQL::allocate(marketplace, error) < 0)
+        int rc = PoolSQL::allocate(marketplace, error);
+
+        rc += PoolSQL::allocate(lxc_marketplace, error);
+
+        if (rc < 0)
         {
             ostringstream oss;
-            oss << "Error trying to create default "
-                << "OpenNebula Systems MarketPlace: " << error;
+            oss << "Error trying to create default marketplaces: " << error;
             NebulaLog::log("MKP", Log::ERROR, oss);
 
             throw runtime_error(oss.str());

src/oca/python/Makefile

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 # Use full path to ensure virtualenv compatibility
-PYTHON = $(shell which python) 
+PYTHON = $(shell which python)
 GDS = $(shell which generateDS)
 PWD = $(shell pwd)
 
@@ -35,7 +35,7 @@ pyone/bindings/__init__.py pyone/bindings/supbind.py: $(schemas)
 	sed -i "s/import sys/import sys\nfrom pyone.util import TemplatedType/" pyone/bindings/__init__.py
 	sed -i "s/(supermod\./(TemplatedType, supermod\./g" pyone/bindings/__init__.py
 
-.PHONY: clean
+.PHONY: clean dist
 clean:
 	rm -rf build dist pyone/bindings *.egg-info doc
 

src/oca/ruby/opennebula/marketplaceapp.rb

@@ -193,6 +193,17 @@ module OpenNebula
                 image = Image.new(Image.build_xml, @client)
                 rc    = image.allocate(tmpl, options[:dsid])
 
+                ds = OpenNebula::Datastore.new_with_id(options[:dsid], @client)
+                image.info
+                ds.info
+
+                xpath = 'TEMPLATE/DRIVER'
+                if ds[xpath] == 'vcenter' && self['FORMAT'] != 'iso' && self['FORMAT'] != 'vmdk'
+                    image.replace({'FORMAT' => 'vmdk'})
+                elsif ds[xpath] && ds[xpath] != 'vcenter' && self['FORMAT'] == 'vmdk'
+                    image.replace({'FORMAT' => ds[xpath] })
+                end
+
                 return { :image => [rc] } if OpenNebula.is_error?(rc)
 
                 image_id = image.id

src/pool/PoolSQL.cc

@@ -148,7 +148,14 @@ int PoolSQL::allocate(PoolObjectSQL *objsql, string& error_str)
 
     if( rc == -1 )
     {
-        _set_lastOID(--lastOID, db, table);
+        lastOID = lastOID - 1;
+
+        if ( lastOID < 0 )
+        {
+            lastOID = 0;
+        }
+
+        _set_lastOID(lastOID, db, table);
     }
 
     unlock();

src/sunstone/OpenNebulaVNC.rb

@@ -109,7 +109,7 @@ class OpenNebulaVNC
 
         @pipe = nil
         @token_folder = File.join(VAR_LOCATION, opts[:token_folder_name])
-        @proxy_path   = File.join(SHARE_LOCATION, "websockify/websocketproxy.py")
+        @proxy_path   = File.join(SHARE_LOCATION, "websockify/run")
         @proxy_port   = config[:vnc_proxy_port]
 
         @proxy_ipv6   = config[:vnc_proxy_ipv6]
@@ -151,10 +151,7 @@ class OpenNebulaVNC
             proxy_options << " -6"
         end
 
-        system("which python2 >/dev/null 2>&1")
-        python = $?.success? ? "python2" : "python"
-
-        cmd ="#{python} #{@proxy_path} #{proxy_options} #{@proxy_port}"
+        cmd ="python #{@proxy_path} #{proxy_options} #{@proxy_port}"
 
         begin
             @logger.info { "Starting VNC proxy: #{cmd}" }

src/sunstone/public/app/tabs/vms-tab/panels/conf.js

@@ -56,7 +56,7 @@ define(function(require) {
     var conf = {};
     var template = this.element.TEMPLATE;
 
-    $.each(["OS", "FEATURES", "INPUT", "GRAPHICS", "RAW", "CONTEXT"], function(){
+    $.each(["OS", "FEATURES", "INPUT", "GRAPHICS", "RAW", "CONTEXT", "CPU_MODEL"], function(){
       if(template[this] != undefined){
         conf[this] = template[this];
       }

src/tm_mad/vcenter/clone

@@ -110,7 +110,9 @@ rescue Exception => e
     message = "Error clone virtual disk #{src_path} in "\
               "datastore #{target_ds_name_vc}. "\
               "Reason: #{e.message}\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/tm_mad/vcenter/cpds

@@ -92,7 +92,9 @@ begin
 rescue Exception => e
     message = "Error copying img #{src_path} to #{target_ds_name_vc} "\
               "Reason: #{e.message}\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/tm_mad/vcenter/delete

@@ -104,6 +104,9 @@ begin
 
 rescue Exception => e
     vi_client.close_connection if vi_client
-    STDERR.puts "#{@error_message}. Reason: #{e.message}\n#{e.backtrace}"
+    message = "#{@error_message}. Reason: #{e.message}\n#{e.backtrace}"
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 end

src/tm_mad/vcenter/mkimage

@@ -88,7 +88,9 @@ begin
 rescue Exception => e
     message = "Error creating virtual disk in #{ds_vc['name']}."\
               " Reason: #{e.message}\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/tm_mad/vcenter/mv

@@ -58,5 +58,7 @@ rescue StandardError => e
               'failed due to '\
               "\"#{e.message}\"\n"
     OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit(-1)
 end

src/tm_mad/vcenter/mvds

@@ -80,7 +80,9 @@ begin
 rescue Exception => e
     message = "Error detaching virtual disk #{disk_id} from vm #{vmid}."\
               " Reason: #{e.message}\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/tm_mad/vcenter/resize

@@ -72,7 +72,9 @@ begin
 rescue Exception => e
     message = "Error resizing disk #{disk_id} for VM #{one_vm["NAME"]} "\
               "Reason: #{e.message}\n#{e.backtrace}"
-    STDERR.puts error_message(message)
+    OpenNebula.log_error(message)
+    STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information]
+
     exit -1
 ensure
     vi_client.close_connection if vi_client

src/vm/VirtualMachine.cc

@@ -2744,7 +2744,8 @@ static std::map<std::string,std::vector<std::string>> UPDATECONF_ATTRS = {
 		   "GUEST_AGENT"} },
 		{"INPUT", {"TYPE", "BUS"} },
 		{"GRAPHICS", {"TYPE", "LISTEN", "PASSWD", "KEYMAP"} },
-		{"RAW", {"TYPE", "DATA", "DATA_VMX"} }
+		{"RAW", {"TYPE", "DATA", "DATA_VMX"} },
+               {"CPU_MODEL", {"MODEL"} }
 	};
 
 /**
@@ -2884,7 +2885,7 @@ int VirtualMachine::updateconf(VirtualMachineTemplate& tmpl, string &err)
     }
 
     // -------------------------------------------------------------------------
-    // Update OS, FEATURES, INPUT, GRAPHICS, RAW
+    // Update OS, FEATURES, INPUT, GRAPHICS, RAW, CPU_MODEL
     // -------------------------------------------------------------------------
     replace_vector_values(obj_template, &tmpl, "OS");
 
@@ -2901,6 +2902,8 @@ int VirtualMachine::updateconf(VirtualMachineTemplate& tmpl, string &err)
 
     replace_vector_values(obj_template, &tmpl, "RAW");
 
+    replace_vector_values(obj_template, &tmpl, "CPU_MODEL");
+
     // -------------------------------------------------------------------------
     // Update CONTEXT: any value
     // -------------------------------------------------------------------------

src/vmm_mad/remotes/lib/lxd/container.rb

@@ -273,7 +273,7 @@ class Container
     # Removes the context section from the LXD configuration and unmap the
     # context device
     def detach_context
-        return unless @one.has_context?
+        return 'no context' unless @one.has_context?
 
         csrc = @lxc['devices']['context']['source'].clone
 

src/vmm_mad/remotes/lib/vcenter_driver/virtual_machine.rb

@@ -1050,7 +1050,7 @@ class VirtualMachine < VCenterDriver::Template
                 key = backing.port.portgroupKey
             end
 
-            @nics[key] = Nic.vc_nic(d)
+            @nics["#{key}#{d.key}"] = Nic.vc_nic(d)
         end
 
         @nics.reject{|k| k == :macs}
@@ -1217,10 +1217,17 @@ class VirtualMachine < VCenterDriver::Template
             if !unmanaged_nics.empty?
                 nics  = get_vcenter_nics
 
-                select = ->(name){
+                select_net =->(ref){
                     device = nil
                     nics.each do |nic|
-                        next unless nic.deviceInfo.summary == name
+                        type = nic.backing.class
+                        if type == NET_CARD
+                            nref = nic.backing.network._ref
+                        else
+                            nref = nic.backing.port.portgroupKey
+                        end
+
+                        next unless nref == ref
                         device = nic
                         break
                     end
@@ -1231,7 +1238,7 @@ class VirtualMachine < VCenterDriver::Template
                 }
 
                 unmanaged_nics.each do |unic|
-                    vnic  = select.call(unic['BRIDGE'])
+                    vnic  = select_net.call(unic['VCENTER_NET_REF'])
                     vcenter_nic_class    = vnic.class
                     new_model = unic['MODEL'] && !unic['MODEL'].empty? && !unic['MODEL'].nil?
                     opennebula_nic_class = nic_model_class(unic['MODEL']) if new_model
@@ -1573,7 +1580,7 @@ class VirtualMachine < VCenterDriver::Template
         end
 
         # grab the last unitNumber to ensure the nic to be added at the end
-        @unic = @unic || get_vcenter_nics.map{|d| d.unitNumber}.max rescue 0
+        @unic = @unic || get_vcenter_nics.map{|d| d.unitNumber}.max || 0
         card_spec = {
             :key => 0,
             :deviceInfo => {

src/vnm_mad/remotes/ovswitch/OpenvSwitch.rb

@@ -391,7 +391,7 @@ private
     def create_bridge
         return if @bridges.keys.include? @nic[:bridge]
 
-        OpenNebula.exec_and_log("#{command(:ovs_vsctl)} add-br #{@nic[:bridge]}")
+        OpenNebula.exec_and_log("#{command(:ovs_vsctl)} --may-exist add-br #{@nic[:bridge]}")
 
         set_bridge_options
 

src/vnm_mad/remotes/vxlan/vxlan.rb

@@ -65,9 +65,11 @@ module VXLAN
             ip_link_conf << "#{option} #{value} "
         end
 
+        # `ip link add ...` returns 2 when vxlan device already exists
+        # allow it to prevent race conditions
         OpenNebula.exec_and_log("#{command(:ip)} link add #{@nic[@attr_vlan_dev]}"\
             " #{mtu} type vxlan id #{@nic[@attr_vlan_id]} #{group} #{ttl}"\
-            " #{tep} #{ip_link_conf}")
+            " #{tep} #{ip_link_conf}", nil, 2)
 
         OpenNebula.exec_and_log("#{command(:ip)} link set #{@nic[@attr_vlan_dev]} up")
     end