stream input from stdin

rather than reading the entire script into memory.

 * Stream STDIN input, running queries a line at a time.
 * Remove MemoryError check, and recommendation for the vendor client.
 * Use CSV/TSV formats with headers for the first line only.
 * Exit with an error code if we are unable to open /dev/tty.
 * Add --noninteractive flag to suppress the destructive-warning prompt
   from the CLI.
 * Commentary on edge cases and followups.

Author: rolandwalker

changelog.md

@@ -6,6 +6,7 @@ Features
 * More complete and up-to-date set of MySQL reserved words for completions.
 * Place exact-leading completions first.
 * Allow history file location to be configured.
+* Stream input to STDIN to consume less memory, and add a `--noninteractive` flag.
 
 
 Bug Fixes

mycli/main.py

@@ -1554,6 +1554,7 @@ def get_last_query(self) -> str | None:
     "--password-file", type=click.Path(), help="File or FIFO path containing the password to connect to the db if not specified otherwise."
 )
 @click.argument("database", default="", nargs=1)
+@click.option("--noninteractive", is_flag=True, help="Don't prompt during batch input.  Recommended.")
 def cli(
     database: str,
     user: str | None,
@@ -1598,6 +1599,7 @@ def cli(
     init_command: str | None,
     charset: str | None,
     password_file: str | None,
+    noninteractive: bool,
 ) -> None:
     """A MySQL terminal client with auto-completion and syntax highlighting.
 
@@ -1844,36 +1846,36 @@ def cli(
         mycli.run_cli()
     else:
         stdin = click.get_text_stream("stdin")
-        try:
-            stdin_text = stdin.read()
-        except MemoryError:
-            click.secho("Failed! Ran out of memory.", err=True, fg="red")
-            click.secho("You might want to try the official mysql client.", err=True, fg="red")
-            click.secho("Sorry... :(", err=True, fg="red")
-            sys.exit(1)
-
-        if mycli.destructive_warning and is_destructive(stdin_text):
+        counter = 0
+        for stdin_text in stdin:
+            if counter:
+                if csv:
+                    mycli.main_formatter.format_name = "csv-noheader"
+                elif not table:
+                    mycli.main_formatter.format_name = "tsv_noheader"
+            else:
+                if csv:
+                    mycli.main_formatter.format_name = "csv"
+                elif not table:
+                    mycli.main_formatter.format_name = "tsv"
+            counter += 1
+            warn_confirmed = True
+            if not noninteractive and mycli.destructive_warning and is_destructive(stdin_text):
+                try:
+                    # this seems to work, even though we are reading from stdin above
+                    sys.stdin = open("/dev/tty")
+                    # bug: the prompt will not be visible if stdout is redirected
+                    warn_confirmed = confirm_destructive_query(stdin_text)
+                except (IOError, OSError):
+                    mycli.logger.warning("Unable to open TTY as stdin.")
+                    sys.exit(1)
             try:
-                sys.stdin = open("/dev/tty")
-                warn_confirmed = confirm_destructive_query(stdin_text)
-            except (IOError, OSError):
-                mycli.logger.warning("Unable to open TTY as stdin.")
-            if not warn_confirmed:
-                sys.exit(0)
-
-        try:
-            new_line = True
-
-            if csv:
-                mycli.main_formatter.format_name = "csv"
-            elif not table:
-                mycli.main_formatter.format_name = "tsv"
-
-            mycli.run_query(stdin_text, new_line=new_line)
-            sys.exit(0)
-        except Exception as e:
-            click.secho(str(e), err=True, fg="red")
-            sys.exit(1)
+                if warn_confirmed:
+                    mycli.run_query(stdin_text, new_line=True)
+            except Exception as e:
+                click.secho(str(e), err=True, fg="red")
+                sys.exit(1)
+        sys.exit(0)
     mycli.close()
 
 

mycli/packages/prompt_utils.py

@@ -25,6 +25,8 @@ def __repr__(self):
 BOOLEAN_TYPE = ConfirmBoolParamType()
 
 
+# todo: all of confirm_destructive_query must be reimplemented in terms
+# of /dev/tty, not stdin/stdout.
 def confirm_destructive_query(queries: str) -> bool | None:
     """Check if the query is destructive and prompts the user to confirm.