diff --git a/aiosmtpd/docs/NEWS.rst b/aiosmtpd/docs/NEWS.rst
index cf7251e0..14728769 100644
--- a/aiosmtpd/docs/NEWS.rst
+++ b/aiosmtpd/docs/NEWS.rst
@@ -14,6 +14,7 @@ Fixed/Improved
 --------------
 * All Controllers now have more rationale design, as they are now composited from a Base + a Mixin
 * A whole bunch of annotations
+* Delay UID change after loop start (Closes #304)
 
 
 1.4.4.post2 (2023-01-19)
diff --git a/aiosmtpd/main.py b/aiosmtpd/main.py
index 166484ca..6cfb7059 100644
--- a/aiosmtpd/main.py
+++ b/aiosmtpd/main.py
@@ -217,22 +217,6 @@ def parseargs(args: Optional[Sequence[str]] = None) -> Tuple[ArgumentParser, Nam
 def main(args: Optional[Sequence[str]] = None) -> None:
     parser, args = parseargs(args=args)
 
-    if args.setuid:  # pragma: on-win32
-        if pwd is None:
-            print(
-                'Cannot import module "pwd"; try running with -n option.',
-                file=sys.stderr,
-            )
-            sys.exit(1)
-        nobody = pwd.getpwnam("nobody").pw_uid
-        try:
-            os.setuid(nobody)
-        except PermissionError:
-            print(
-                'Cannot setuid "nobody"; try running with -n option.', file=sys.stderr
-            )
-            sys.exit(1)
-
     if args.tlscert and args.tlskey:
         tls_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
         tls_context.check_hostname = False
@@ -279,6 +263,24 @@ def main(args: Optional[Sequence[str]] = None) -> None:
     log.debug(f"server_loop = {server_loop}")
     log.info("Server is listening on %s:%s", args.host, args.port)
 
+    # Change the UID after opening the port. This allows listening
+    # on port < 1024 without any system tweak.
+    if args.setuid:  # pragma: on-win32
+        if pwd is None:
+            print(
+                'Cannot import module "pwd"; try running with -n option.',
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        nobody = pwd.getpwnam("nobody").pw_uid
+        try:
+            os.setuid(nobody)
+        except PermissionError:
+            print(
+                'Cannot setuid "nobody"; try running with -n option.', file=sys.stderr
+            )
+            sys.exit(1)
+
     # Signal handlers are only supported on *nix, so just ignore the failure
     # to set this on Windows.
     with suppress(NotImplementedError):