F #1473: Update SSH config and sockets (#4733)

* F #1473: Update SSH config and sockets

- install files via install.sh script
- replace crontab with systemd timer
- update opennebula.service dependency
- move ssh master sockets from ~oneadmin to /run/one/ssh-socks
- modify ssh wrapper to not use external programs but only bash builtins
- update ssh configs

Signed-off-by: Petr Ospalý <pospaly@opennebula.io>

* F #1473: Update SSH config and sockets

* F #1473: Update SSH config and sockets

* F #1473: Update SSH wrapper

- rename SSH_USE_MASTER_SOCK to SSH_OPT_CONTROL_MASTER
- change the logic of this directive to be a value of the SSH
  'ControlMaster' option (default is 'auto')
- delete the SSH_USE_MASTER_SOCK from opennebula.service (not needed
  anymore)

Signed-off-by: Petr Ospalý <pospaly@opennebula.io>

Co-authored-by: Vlastimil Holer <vholer@opennebula.io>

install.sh

@@ -231,6 +231,7 @@ SHARE_DIRS="$SHARE_LOCATION/examples \
             $SHARE_LOCATION/websockify/websockify \
             $SHARE_LOCATION/esx-fw-vnc \
             $SHARE_LOCATION/oneprovision \
+            $SHARE_LOCATION/ssh \
             $SHARE_LOCATION/start-scripts"
 
 ETC_DIRS="$ETC_LOCATION/vmm_exec \
@@ -257,6 +258,7 @@ LIB_DIRS="$LIB_LOCATION/ruby \
           $LIB_LOCATION/ruby/vendors \
           $LIB_LOCATION/mads \
           $LIB_LOCATION/sh \
+          $LIB_LOCATION/sh/override \
           $LIB_LOCATION/ruby/cli \
           $LIB_LOCATION/ruby/cli/one_helper \
           $LIB_LOCATION/ruby/vcenter_driver \
@@ -667,6 +669,9 @@ INSTALL_FILES=(
     ONE_CLI_LIB_FILES:$LIB_LOCATION/ruby/cli/one_helper
     VENDOR_DIRS:$LIB_LOCATION/ruby/vendors
     START_SCRIPT_SHARE_FILES:$SHARE_LOCATION/start-scripts
+    SSH_SH_LIB_FILES:$LIB_LOCATION/sh
+    SSH_SH_OVERRIDE_LIB_FILES:$LIB_LOCATION/sh/override
+    SSH_SHARE_FILES:$SHARE_LOCATION/ssh
 )
 
 INSTALL_CLIENT_FILES=(
@@ -2665,6 +2670,17 @@ ONEHEM_ETC_FILES="src/hem/etc/onehem-server.conf"
 
 DOCKER_MACHINE_BIN_FILES="src/docker_machine/src/docker_machine/bin/docker-machine-driver-opennebula"
 
+#-----------------------------------------------------------------------------
+# SSH files
+#-----------------------------------------------------------------------------
+
+SSH_SH_LIB_FILES="share/ssh/bin/ssh-socks-cleaner"
+
+SSH_SH_OVERRIDE_LIB_FILES="share/ssh/bin/ssh"
+
+SSH_SHARE_FILES="share/ssh/etc/config \
+                 share/ssh/etc/config-pre7.6"
+
 #-----------------------------------------------------------------------------
 # MAN files
 #-----------------------------------------------------------------------------

share/etc/cron.d/opennebula-ssh-socks-cleaner

@@ -1,15 +0,0 @@
-SHELL=/bin/sh
-
-#####################################################################
-# WARNING: This cronjob command parameters must be aligned with
-# 'ControlPath' and 'ControlPersist' options in oneadmin's SSH
-# configuration (/var/lib/one/.ssh/config) otherwise it will
-# not have desired effect!
-#####################################################################
-
-# Cleans all oneadmin's SSH master sockets for the persistent
-# connections when they reach their age of one minute as workaround to
-# race condition in OpenSSH during close of the master socket.
-# Possibly related to https://bugzilla.mindrot.org/show_bug.cgi?id=3067
-
-* * * * * oneadmin find ~oneadmin -maxdepth 1 -type s -name '.ssh-M-*.sock' -mmin +1 -delete

share/pkgs/services/systemd/opennebula-ssh-socks-cleaner.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=OpenNebula SSH persistent connection cleaner
+
+[Service]
+Group=oneadmin
+User=oneadmin
+Type=oneshot
+ExecStart=/usr/lib/one/sh/ssh-socks-cleaner

share/pkgs/services/systemd/opennebula-ssh-socks-cleaner.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=OpenNebula SSH persistent connection cleaner
+After=remote-fs.target
+
+[Timer]
+OnActiveSec=0s
+OnUnitActiveSec=30s
+AccuracySec=1s
+
+[Install]
+WantedBy=default.target

share/pkgs/services/systemd/opennebula.service

@@ -5,13 +5,13 @@ After=mariadb.service mysql.service
 After=opennebula-ssh-agent.service
 Wants=opennebula-scheduler.service opennebula-hem.service
 Wants=opennebula-ssh-agent.service
+Wants=opennebula-ssh-socks-cleaner.timer
 
 [Service]
 Type=notify
 Group=oneadmin
 User=oneadmin
 Environment="PATH=/usr/lib/one/sh/override:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-Environment="SSH_USE_MASTER_SOCK=yes"
 EnvironmentFile=-/var/run/one/ssh-agent.env
 ExecStartPre=-/usr/sbin/logrotate -f /etc/logrotate.d/opennebula -s /var/lib/one/.logrotate.status
 ExecStart=/usr/bin/oned -f

share/pkgs/tmpfiles/opennebula.conf

@@ -1,2 +1,3 @@
 d /var/lock/one 0755 oneadmin oneadmin -
 d /var/run/one 0755 oneadmin oneadmin -
+d /var/run/one/ssh-socks 0700 oneadmin oneadmin -

share/ssh/bin/ssh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # -------------------------------------------------------------------------- #
 # Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                #
@@ -20,8 +20,10 @@
 # for the persistent connections - it will add '-o ControlMaster=auto' to the
 # ssh command in some specific cases (where it does not hang the SSH session).
 #
-# NOTE: It can be disabled by setting the env. var 'SSH_USE_MASTER_SOCK' to
-# 'no' - e.g.: by overriding the systemd unit 'opennebula.service'.
+# NOTE: It can be overriden by setting the env. var 'SSH_OPT_CONTROL_MASTER'
+# either directly when run manually in the shell or in the systemd unit
+# 'opennebula.service' (e.g.: Environment="SSH_OPT_CONTROL_MASTER=no").
+# In either case it must be a valid value for SSH's 'ControlMaster' option...
 
 #
 # global variables
@@ -30,6 +32,9 @@
 # fix the PATH again...
 PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
+# default 'ControlMaster' override value
+SSH_OPT_CONTROL_MASTER="${SSH_OPT_CONTROL_MASTER:-auto}"
+
 
 #
 # functions
@@ -37,6 +42,9 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 is_control_master_used()
 {
+    # match regardless of the case
+    shopt -s nocasematch
+
     # parse the arguments
     _state=nil
     while [ "$#" -gt 0 ] ; do
@@ -47,8 +55,8 @@ is_control_master_used()
                         _state=option
                         ;;
                     -o*)
-                        if echo "$1" | grep -iq -- '^-o[[:space:]]*ControlMaster' ; then
-                            # ControlMaster is already used!
+                        if [[ "$1" =~ ^-o[[:space:]]*ControlMaster.*$ ]] ; then
+                            # ControlMaster option was found!
                             return 0
                         fi
                         _state=nil
@@ -59,8 +67,8 @@ is_control_master_used()
                 esac
                 ;;
             option)
-                if echo "$1" | grep -iq -- '^[[:space:]]*ControlMaster' ; then
-                    # ControlMaster is already used!
+                if [[ "$1" =~ ^[[:space:]]*ControlMaster.*$ ]] ; then
+                    # ControlMaster option was found!
                     return 0
                 fi
                 _state=nil
@@ -69,6 +77,9 @@ is_control_master_used()
         shift
     done
 
+    # revert back to normal
+    shopt -u nocasematch
+
     # ControlMaster was not encountered
     return 1
 }
@@ -78,20 +89,10 @@ is_control_master_used()
 # main
 #
 
-case "$SSH_USE_MASTER_SOCK" in
-    yes|YES|Yes|1|true|True|TRUE)
-        : 'continue below'
-        ;;
-    *)
-        # no override and pass the command without changes to the ssh
-        exec ssh "$@"
-        ;;
-esac
-
 if is_control_master_used "$@" ; then
     # do not override anything - ControlMaster was already set explicitly
     exec ssh "$@"
 fi
 
 # now we can override the ssh command with the enabled master socket
-exec ssh -o ControlMaster=auto "$@"
+exec ssh -o "ControlMaster=${SSH_OPT_CONTROL_MASTER}" "$@"

share/ssh/bin/ssh-socks-cleaner

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+##############################################################################
+# WARNING: The arguments of the 'find' command below must be aligned with
+# 'ControlPath' and 'ControlPersist' options in oneadmin's SSH configuration
+# (/var/lib/one/.ssh/config) otherwise it will not have the desired effect!
+##############################################################################
+
+# This script enforce the cleanup of all oneadmin's SSH master sockets for the
+# persistent connections when they reach their age of one minute. It as a
+# workaround to a race condition inside the OpenSSH which occurs during the
+# closing of the master sockets.
+# Possibly related to: https://bugzilla.mindrot.org/show_bug.cgi?id=3067
+
+find /var/run/one/ssh-socks \
+    -maxdepth 1 \
+    -type s \
+    -name 'ctl-M-*.sock' \
+    -mmin +1 \
+    -print | while read -r sockname ; do
+        # atomic operation - no other ssh client should be disrupted
+        mv -f "$sockname" "$sockname"~todelete
+
+        # stop the multiplexing ('this' is just unnecessary gibberish)
+        ssh -S "$sockname"~todelete -O stop this </dev/null >/dev/null
+
+        # delete the old socket
+        rm -f "$sockname"~todelete
+    done

share/ssh/etc/config

@@ -12,16 +12,17 @@ Host *
   #############################################################################
   # 'ControlMaster' is overriden by OpenNebula's drivers when needed
   ControlMaster no
-  # The following options must be aligned with the accompanying cronjob:
-  # /etc/cron.d/opennebula-ssh-socks-cleaner (if present) which implements
-  # workaround for OpenSSH race condition during the closing of the master
-  # socket.
+  # The following options must be aligned with the accompanying timer/cronjob:
+  # opennebula-ssh-socks-cleaner (if present) which implements workaround for
+  # OpenSSH race condition during the closing of the master socket.
   #
-  # 'ControlPersist' cannot be set less than 120 (two minutes) to offset the
-  # minimum one minute time before each cronjob check! It will also not change
-  # behavior even if it set much higher or to infinity (0).
+  # 'ControlPersist' should be set to more than twice the period after which
+  # timer or cronjob is run - to offset the delay - e.g.: timer job is run each
+  # 30s then 'ControlPersist' should be at least one minute. It will also not
+  # change the behavior even if it set much higher or to the infinity (0) - it
+  # is limited by the timer/cronjob *AND* the command which is executed inside.
   #
-  # (+) Add another 10s to give crond a room for cleanup
-  ControlPersist 130s
-  # 'ControlPath' must match the 'find' command in the cronjob above!
-  ControlPath ~/.ssh-M-%C.sock
+  # (+) Add another 10s to give timer/cronjob a room for cleanup
+  ControlPersist 70s
+  # 'ControlPath' must be in-sync with the script run by timer/cronjob above!
+  ControlPath /run/one/ssh-socks/ctl-M-%C.sock

share/ssh/etc/config-pre7.6

@@ -18,24 +18,25 @@ Match !exec "ssh-keygen -F %h 2>/dev/null || ssh-keygen -F %h -f /etc/ssh/ssh_kn
   #############################################################################
   # 'ControlMaster' is overriden by OpenNebula's drivers when needed
   ControlMaster no
-  # The following options must be aligned with the accompanying cronjob:
-  # /etc/cron.d/opennebula-ssh-socks-cleaner (if present) which implements
-  # workaround for OpenSSH race condition during the closing of the master
-  # socket.
+  # The following options must be aligned with the accompanying timer/cronjob:
+  # opennebula-ssh-socks-cleaner (if present) which implements workaround for
+  # OpenSSH race condition during the closing of the master socket.
   #
-  # 'ControlPersist' cannot be set less than 120 (two minutes) to offset the
-  # minimum one minute time before each cronjob check! It will also not change
-  # behavior even if it set much higher or to infinity (0).
+  # 'ControlPersist' should be set to more than twice the period after which
+  # timer or cronjob is run - to offset the delay - e.g.: timer job is run each
+  # 30s then 'ControlPersist' should be at least one minute. It will also not
+  # change the behavior even if it set much higher or to the infinity (0) - it
+  # is limited by the timer/cronjob *AND* the command which is executed inside.
   #
-  # (+) Add another 10s to give crond a room for cleanup
-  ControlPersist 130s
-  # 'ControlPath' must match the 'find' command in the cronjob above!
-  ControlPath ~/.ssh-M-%C.sock
+  # (+) Add another 10s to give timer/cronjob a room for cleanup
+  ControlPersist 70s
+  # 'ControlPath' must be in-sync with the script run by timer/cronjob above!
+  ControlPath /run/one/ssh-socks/ctl-M-%C.sock
 
 Host *
   StrictHostKeyChecking yes
   ServerAliveInterval 10
   # IMPORTANT: set the following 'Control*' options the same way as above
   ControlMaster no
-  ControlPersist 130s
-  ControlPath ~/.ssh-M-%C.sock
+  ControlPersist 70s
+  ControlPath /run/one/ssh-socks/ctl-M-%C.sock