terminal-util: don't assume errno is correctly set when using isatty_safe()

let's instead generate ENOTTY on our own. This is more correct with out
coding style (since we generally do not propagate errors via errno), and
also addresses #34039 as side effect. (#34039 really needs to be fixed
in musl though, too, this is just a work-around as side-effect).

Fixes: #34039

src/basic/terminal-util.c

@@ -287,7 +287,7 @@ int open_terminal(const char *name, int mode) {
         }
 
         if (!isatty_safe(fd))
-                return negative_errno();
+                return -ENOTTY;
 
         return TAKE_FD(fd);
 }
@@ -1508,7 +1508,7 @@ int vt_restore(int fd) {
         assert(fd >= 0);
 
         if (!isatty_safe(fd))
-                return log_debug_errno(errno, "Asked to restore the VT for an fd that does not refer to a terminal: %m");
+                return log_debug_errno(SYNTHETIC_ERRNO(ENOTTY), "Asked to restore the VT for an fd that does not refer to a terminal: %m");
 
         if (ioctl(fd, KDSETMODE, KD_TEXT) < 0)
                 RET_GATHER(ret, log_debug_errno(errno, "Failed to set VT to text mode, ignoring: %m"));
@@ -1535,7 +1535,7 @@ int vt_release(int fd, bool restore) {
          * VT-switching modes. */
 
         if (!isatty_safe(fd))
-                return log_debug_errno(errno, "Asked to release the VT for an fd that does not refer to a terminal: %m");
+                return log_debug_errno(SYNTHETIC_ERRNO(ENOTTY), "Asked to release the VT for an fd that does not refer to a terminal: %m");
 
         if (ioctl(fd, VT_RELDISP, 1) < 0)
                 return -errno;

src/getty-generator/getty-generator.c

@@ -95,7 +95,7 @@ static int verify_tty(const char *name) {
                 return -errno;
 
         if (!isatty_safe(fd))
-                return -errno;
+                return -ENOTTY;
 
         return 0;
 }

src/shared/wall.c

@@ -32,7 +32,7 @@ static int write_to_terminal(const char *tty, const char *message) {
                 return -errno;
 
         if (!isatty_safe(fd))
-                return -errno;
+                return -ENOTTY;
 
         return loop_write_full(fd, message, SIZE_MAX, TIMEOUT_USEC);
 }