F #1473: Fix and improve SSH config (#4713)

- workaround issues with persistent connections
- add new cronjob

Co-authored-by: Vlastimil Holer <vholer@opennebula.io>

share/etc/cron.d/opennebula-node

@@ -1,4 +1,4 @@
-SHELL=/bin/bash
+SHELL=/bin/sh
 
 # activate LVM volumes on reboot
 @reboot oneadmin test -x /var/tmp/one/tm/fs_lvm/activate && /var/tmp/one/tm/fs_lvm/activate >/dev/null 2>&1

share/etc/cron.d/opennebula-ssh-socks-cleaner

@@ -0,0 +1,15 @@
+SHELL=/bin/sh
+
+#####################################################################
+# WARNING: This cronjob command parameters must be aligned with
+# 'ControlPath' and 'ControlPersist' options in oneadmin's SSH
+# configuration (/var/lib/one/.ssh/config) otherwise it will
+# not have desired effect!
+#####################################################################
+
+# Cleans all oneadmin's SSH master sockets for the persistent
+# connections when they reach their age of one minute as workaround to
+# race condition in OpenSSH during close of the master socket.
+# Possibly related to https://bugzilla.mindrot.org/show_bug.cgi?id=3067
+
+* * * * * oneadmin find ~oneadmin -maxdepth 1 -type s -name '.ssh-M-*.sock' -mmin +1 -delete

share/pkgs/ssh/config

@@ -7,8 +7,19 @@
 #####################################################################
 
 Host *
-  # since SSH version 7.6+
   StrictHostKeyChecking accept-new
+  ServerAliveInterval 10
   ControlMaster auto
-  ControlPath ~/.ssh-%C
-  ControlPersist 5s
+  #####################################################################
+  # The following options must be aligned with the accompanying cronjob
+  # /etc/cron.d/opennebula-ssh-socks-cleaner which implements workaround
+  # for OpenSSH race condition during close of the master socket.
+  #
+  # 'ControlPersist' cannot be set less than 120 (two minutes) to offset the
+  # minimum one minute time before each cronjob check! It will also not change
+  # behavior even if it set much higher or to infinity (0).
+  #
+  # (+) Add another 10s to give crond a room for cleanup
+  ControlPersist 130s
+  # 'ControlPath' must match the 'find' command in the cronjob above!
+  ControlPath ~/.ssh-M-%C.sock

share/pkgs/ssh/config-pre7.6

@@ -14,12 +14,26 @@
 # All further accesses already use strict host key checking.
 Match !exec "ssh-keygen -F %h 2>/dev/null || ssh-keygen -F %h -f /etc/ssh/ssh_known_hosts 2>/dev/null"
   StrictHostKeyChecking no
+  ServerAliveInterval 10
   ControlMaster auto
-  ControlPath ~/.ssh-%C
-  ControlPersist 5s
+  #####################################################################
+  # The following options must be aligned with the accompanying cronjob
+  # /etc/cron.d/opennebula-ssh-socks-cleaner which implements workaround
+  # for OpenSSH race condition during close of the master socket.
+  #
+  # 'ControlPersist' cannot be set less than 120 (two minutes) to offset the
+  # minimum one minute time before each cronjob check! It will also not change
+  # behavior even if it set much higher or to infinity (0).
+  #
+  # (+) Add another 10s to give crond a room for cleanup
+  ControlPersist 130s
+  # 'ControlPath' must match the 'find' command in the cronjob above!
+  ControlPath ~/.ssh-M-%C.sock
 
 Host *
   StrictHostKeyChecking yes
+  ServerAliveInterval 10
   ControlMaster auto
-  ControlPath ~/.ssh-%C
-  ControlPersist 5s
+  # IMPORTANT: set the following two options the same way as above
+  ControlPersist 130s
+  ControlPath ~/.ssh-M-%C.sock