Cygwin: console: tty::restore really restores the previous mode

Previously, tty::restore sets the console mode to a predetermined
console mode that is widely common for many non-cygwin console apps.
So, if a non-cygwin app that is started from cygwin process changes
the console mode and executes cygwin sub-process, the console mode
is changed to the predetermined mode rather than being restored the
original mode that is set by the non-cygwin app.
With this patch, the console mode is stored when a cygwin process is
started from non-cygwin app, then tty::restore restores the previous
console mode that is used by the previous non-cygwin app when the
cygwin app exits.

Addresses: msys2/msys2-runtime#268
Fixes: 3312f2d ("Cygwin: console: Redesign mode set strategy on close().")
Reported-by: Eu Pin Tien, Jeremy Drake <[email protected]>
Signed-off-by: Takashi Yano <[email protected]>
Signed-off-by: Eu-Pin Tien <[email protected]>
Signed-off-by: Johannes Schindelin <[email protected]>

Author: tyan0

winsup/cygwin/fhandler/console.cc

@@ -804,6 +804,9 @@ fhandler_console::rabuflen ()
   return con_ra.rabuflen;
 }
 
+static DWORD prev_input_mode_backup;
+static DWORD prev_output_mode_backup;
+
 /* The function set_{in,out}put_mode() should be static so that they
    can be called even after the fhandler_console instance is deleted. */
 void
@@ -818,11 +821,11 @@ fhandler_console::set_input_mode (tty::cons_mode m, const termios *t,
   GetConsoleMode (p->input_handle, &oflags);
   DWORD flags = oflags
     & (ENABLE_EXTENDED_FLAGS | ENABLE_INSERT_MODE | ENABLE_QUICK_EDIT_MODE);
-  con.curr_input_mode = m;
   switch (m)
     {
     case tty::restore:
-      flags |= ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT;
+      flags = con.prev_input_mode;
+      con.prev_input_mode = prev_input_mode_backup;
       break;
     case tty::cygwin:
       flags |= ENABLE_WINDOW_INPUT;
@@ -846,6 +849,12 @@ fhandler_console::set_input_mode (tty::cons_mode m, const termios *t,
 	flags |= ENABLE_PROCESSED_INPUT;
       break;
     }
+  if (con.curr_input_mode != tty::cygwin && m == tty::cygwin)
+    {
+      prev_input_mode_backup = con.prev_input_mode;
+      con.prev_input_mode = oflags;
+    }
+  con.curr_input_mode = m;
   SetConsoleMode (p->input_handle, flags);
   if (!(oflags & ENABLE_VIRTUAL_TERMINAL_INPUT)
       && (flags & ENABLE_VIRTUAL_TERMINAL_INPUT)
@@ -868,10 +877,11 @@ fhandler_console::set_output_mode (tty::cons_mode m, const termios *t,
   if (con.orig_virtual_terminal_processing_mode)
     flags |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
   WaitForSingleObject (p->output_mutex, mutex_timeout);
-  con.curr_output_mode = m;
   switch (m)
     {
     case tty::restore:
+      flags = con.prev_output_mode;
+      con.prev_output_mode = prev_output_mode_backup;
       break;
     case tty::cygwin:
       if (wincap.has_con_24bit_colors () && !con_is_legacy)
@@ -883,6 +893,12 @@ fhandler_console::set_output_mode (tty::cons_mode m, const termios *t,
 	flags |= DISABLE_NEWLINE_AUTO_RETURN;
       break;
     }
+  if (con.curr_output_mode != tty::cygwin && m == tty::cygwin)
+    {
+      prev_output_mode_backup = con.prev_output_mode;
+      GetConsoleMode (p->output_handle, &con.prev_output_mode);
+    }
+  con.curr_output_mode = m;
   acquire_attach_mutex (mutex_timeout);
   DWORD resume_pid = attach_console (con.owner);
   SetConsoleMode (p->output_handle, flags);
@@ -916,7 +932,7 @@ fhandler_console::cleanup_for_non_cygwin_app (handle_set_t *p)
   /* Cleaning-up console mode for non-cygwin app. */
   /* conmode can be tty::restore when non-cygwin app is
      exec'ed from login shell. */
-  tty::cons_mode conmode = cons_mode_on_close (p);
+  tty::cons_mode conmode = cons_mode_on_close ();
   set_output_mode (conmode, ti, p);
   set_input_mode (conmode, ti, p);
   set_disable_master_thread (con.owner == GetCurrentProcessId ());
@@ -1978,9 +1994,8 @@ fhandler_console::close (int flag)
   if (shared_console_info[unit] && myself->ppid == 1
       && (dev_t) myself->ctty == get_device ())
     {
-      tty::cons_mode conmode = cons_mode_on_close (&handle_set);
-      set_output_mode (conmode, &get_ttyp ()->ti, &handle_set);
-      set_input_mode (conmode, &get_ttyp ()->ti, &handle_set);
+      set_output_mode (tty::restore, &get_ttyp ()->ti, &handle_set);
+      set_input_mode (tty::restore, &get_ttyp ()->ti, &handle_set);
       set_disable_master_thread (true, this);
     }
 
@@ -4687,26 +4702,10 @@ fhandler_console::fstat (struct stat *st)
 }
 
 tty::cons_mode
-fhandler_console::cons_mode_on_close (handle_set_t *p)
+fhandler_console::cons_mode_on_close ()
 {
-  const _minor_t unit = p->unit;
-
   if (myself->ppid != 1) /* Execed from normal cygwin process. */
     return tty::cygwin;
 
-  if (!process_alive (con.owner)) /* The Master process already died. */
-    return tty::restore;
-  if (con.owner == GetCurrentProcessId ()) /* Master process */
-    return tty::restore;
-
-  PROCESS_BASIC_INFORMATION pbi;
-  NTSTATUS status =
-    NtQueryInformationProcess (GetCurrentProcess (), ProcessBasicInformation,
-			       &pbi, sizeof (pbi), NULL);
-  if (NT_SUCCESS (status)
-      && con.owner == (DWORD) pbi.InheritedFromUniqueProcessId)
-    /* The parent is the stub process. */
-    return tty::restore;
-
-  return tty::native; /* Otherwise */
+  return tty::restore; /* otherwise, restore */
 }

winsup/cygwin/local_includes/fhandler.h

@@ -2146,6 +2146,8 @@ class dev_console
   bool disable_master_thread;
   tty::cons_mode curr_input_mode;
   tty::cons_mode curr_output_mode;
+  DWORD prev_input_mode;
+  DWORD prev_output_mode;
   bool master_thread_suspended;
   int num_processed; /* Number of input events in the current input buffer
 			already processed by cons_master_thread(). */
@@ -2366,7 +2368,7 @@ class fhandler_console: public fhandler_termios
 
   void setup_pcon_hand_over ();
   static void pcon_hand_over_proc ();
-  static tty::cons_mode cons_mode_on_close (handle_set_t *);
+  static tty::cons_mode cons_mode_on_close ();
 
   friend tty_min * tty_list::get_cttyp ();
 };

winsup/cygwin/release/3.6.1

@@ -0,0 +1,5 @@
+Fixes:
+------
+
+- Console mode is really restored to the previous mode.
+  Addresses: https://github.com/msys2/msys2-runtime/issues/268