diff --git a/pygmt/helpers/utils.py b/pygmt/helpers/utils.py
index 8997a2b0df1..dd202d2e840 100644
--- a/pygmt/helpers/utils.py
+++ b/pygmt/helpers/utils.py
@@ -350,7 +350,13 @@ def build_arg_list(
             gmt_args = [str(infile), *gmt_args]
         else:
             gmt_args = [str(_file) for _file in infile] + gmt_args
-    if outfile:
+    if outfile is not None:
+        if (
+            not isinstance(outfile, str | pathlib.PurePath)
+            or str(outfile) in {"", ".", ".."}
+            or str(outfile).endswith(("/", "\\"))
+        ):
+            raise GMTInvalidInput(f"Invalid output file name '{outfile}'.")
         gmt_args.append(f"->{outfile}")
     return gmt_args
 
diff --git a/pygmt/tests/test_helpers.py b/pygmt/tests/test_helpers.py
index ea9e4c87225..ea966753535 100644
--- a/pygmt/tests/test_helpers.py
+++ b/pygmt/tests/test_helpers.py
@@ -12,6 +12,7 @@
 from pygmt.helpers import (
     GMTTempFile,
     args_in_kwargs,
+    build_arg_list,
     data_kind,
     kwargs_to_strings,
     unique_name,
@@ -137,6 +138,18 @@ def test_gmttempfile_read():
         assert tmpfile.read(keep_tabs=True) == "in.dat: N = 2\t<1/3>\t<2/4>\n"
 
 
+@pytest.mark.parametrize(
+    "outfile",
+    [123, "", ".", "..", "path/to/dir/", "path\\to\\dir\\", Path(), Path("..")],
+)
+def test_build_arg_list_invalid_output(outfile):
+    """
+    Test that build_arg_list raises an exception when output file name is invalid.
+    """
+    with pytest.raises(GMTInvalidInput):
+        build_arg_list({}, outfile=outfile)
+
+
 def test_args_in_kwargs():
     """
     Test that args_in_kwargs function returns correct Boolean responses.