ENG-7746: include assets in backend.zip (#5814)

Author: adhami3310

reflex/reflex.py

@@ -422,13 +422,21 @@ def compile(dry: bool, rich: bool):
     default=constants.Env.PROD.value,
     help="The environment to export the app in.",
 )
+@click.option(
+    "--exclude-from-backend",
+    "backend_excluded_dirs",
+    multiple=True,
+    type=click.Path(exists=True, path_type=Path, resolve_path=True),
+    help="Files or directories to exclude from the backend zip. Can be used multiple times.",
+)
 def export(
     zip: bool,
     frontend_only: bool,
     backend_only: bool,
     zip_dest_dir: str,
     upload_db_file: bool,
     env: LITERAL_ENV,
+    backend_excluded_dirs: tuple[Path, ...] = (),
 ):
     """Export the app to a zip file."""
     from reflex.utils import export as export_utils
@@ -455,6 +463,7 @@ def export(
         upload_db_file=upload_db_file,
         env=constants.Env.DEV if env == constants.Env.DEV else constants.Env.PROD,
         loglevel=config.loglevel.subprocess_level(),
+        backend_excluded_dirs=backend_excluded_dirs,
     )
 
 
@@ -660,6 +669,13 @@ def makemigrations(message: str | None):
     "--config",
     help="path to the config file",
 )
+@click.option(
+    "--exclude-from-backend",
+    "backend_excluded_dirs",
+    multiple=True,
+    type=click.Path(exists=True, path_type=Path, resolve_path=True),
+    help="Files or directories to exclude from the backend zip. Can be used multiple times.",
+)
 def deploy(
     app_name: str | None,
     app_id: str | None,
@@ -673,6 +689,7 @@ def deploy(
     project_name: str | None,
     token: str | None,
     config_path: str | None,
+    backend_excluded_dirs: tuple[Path, ...] = (),
 ):
     """Deploy the app to the Reflex hosting service."""
     from reflex_cli.utils import dependency
@@ -721,6 +738,7 @@ def deploy(
                 zipping=zipping,
                 loglevel=config.loglevel.subprocess_level(),
                 upload_db_file=upload_db,
+                backend_excluded_dirs=backend_excluded_dirs,
             )
         ),
         regions=list(region),

reflex/utils/build.py

@@ -26,64 +26,77 @@ def set_env_json():
 
 
 def _zip(
+    *,
     component_name: constants.ComponentName,
-    target: str | Path,
-    root_dir: str | Path,
-    exclude_venv_dirs: bool,
-    upload_db_file: bool = False,
-    dirs_to_exclude: set[str] | None = None,
-    files_to_exclude: set[str] | None = None,
-    top_level_dirs_to_exclude: set[str] | None = None,
+    target: Path,
+    root_directory: Path,
+    exclude_venv_directories: bool,
+    include_db_file: bool = False,
+    directory_names_to_exclude: set[str] | None = None,
+    files_to_exclude: set[Path] | None = None,
     globs_to_include: list[str] | None = None,
 ) -> None:
     """Zip utility function.
 
     Args:
         component_name: The name of the component: backend or frontend.
         target: The target zip file.
-        root_dir: The root directory to zip.
-        exclude_venv_dirs: Whether to exclude venv directories.
-        upload_db_file: Whether to include local sqlite db files.
-        dirs_to_exclude: The directories to exclude.
+        root_directory: The root directory to zip.
+        exclude_venv_directories: Whether to exclude venv directories.
+        include_db_file: Whether to include local sqlite db files.
+        directory_names_to_exclude: The directory names to exclude.
         files_to_exclude: The files to exclude.
-        top_level_dirs_to_exclude: The top level directory names immediately under root_dir to exclude. Do not exclude folders by these names further in the sub-directories.
-        globs_to_include: Apply these globs from the root_dir and always include them in the zip.
+        globs_to_include: Apply these globs from the root_directory and always include them in the zip.
 
     """
     target = Path(target)
-    root_dir = Path(root_dir)
-    dirs_to_exclude = dirs_to_exclude or set()
+    root_directory = Path(root_directory).resolve()
+    directory_names_to_exclude = directory_names_to_exclude or set()
     files_to_exclude = files_to_exclude or set()
-    files_to_zip: list[str] = []
+    files_to_zip: list[Path] = []
     # Traverse the root directory in a top-down manner. In this traversal order,
     # we can modify the dirs list in-place to remove directories we don't want to include.
-    for root, dirs, files in os.walk(root_dir, topdown=True, followlinks=True):
-        root = Path(root)
+    for directory_path, subdirectories_names, subfiles_names in os.walk(
+        root_directory, topdown=True, followlinks=True
+    ):
+        directory_path = Path(directory_path).resolve()
         # Modify the dirs in-place so excluded and hidden directories are skipped in next traversal.
-        dirs[:] = [
-            d
-            for d in dirs
-            if (basename := Path(d).resolve().name) not in dirs_to_exclude
-            and not basename.startswith(".")
-            and (not exclude_venv_dirs or not _looks_like_venv_dir(root / d))
+        subdirectories_names[:] = [
+            subdirectory_name
+            for subdirectory_name in subdirectories_names
+            if subdirectory_name not in directory_names_to_exclude
+            and not any(
+                (directory_path / subdirectory_name).samefile(exclude)
+                for exclude in files_to_exclude
+                if exclude.exists()
+            )
+            and not subdirectory_name.startswith(".")
+            and (
+                not exclude_venv_directories
+                or not _looks_like_venv_directory(directory_path / subdirectory_name)
+            )
         ]
-        # If we are at the top level with root_dir, exclude the top level dirs.
-        if top_level_dirs_to_exclude and root == root_dir:
-            dirs[:] = [d for d in dirs if d not in top_level_dirs_to_exclude]
         # Modify the files in-place so the hidden files and db files are excluded.
-        files[:] = [
-            f
-            for f in files
-            if not f.startswith(".") and (upload_db_file or not f.endswith(".db"))
+        subfiles_names[:] = [
+            subfile_name
+            for subfile_name in subfiles_names
+            if not subfile_name.startswith(".")
+            and (include_db_file or not subfile_name.endswith(".db"))
         ]
         files_to_zip += [
-            str(root / file) for file in files if file not in files_to_exclude
+            directory_path / subfile_name
+            for subfile_name in subfiles_names
+            if not any(
+                (directory_path / subfile_name).samefile(excluded_file)
+                for excluded_file in files_to_exclude
+                if excluded_file.exists()
+            )
         ]
     if globs_to_include:
         for glob in globs_to_include:
             files_to_zip += [
-                str(file)
-                for file in root_dir.glob(glob)
+                file
+                for file in root_directory.glob(glob)
                 if file.name not in files_to_exclude
             ]
     # Create a progress bar for zipping the component.
@@ -100,56 +113,57 @@ def _zip(
         for file in files_to_zip:
             console.debug(f"{target}: {file}", progress=progress)
             progress.advance(task)
-            zipf.write(file, Path(file).relative_to(root_dir))
+            zipf.write(file, Path(file).relative_to(root_directory))
 
 
 def zip_app(
     frontend: bool = True,
     backend: bool = True,
     zip_dest_dir: str | Path | None = None,
-    upload_db_file: bool = False,
+    include_db_file: bool = False,
+    backend_excluded_dirs: tuple[Path, ...] = (),
 ):
     """Zip up the app.
 
     Args:
         frontend: Whether to zip up the frontend app.
         backend: Whether to zip up the backend app.
         zip_dest_dir: The directory to export the zip file to.
-        upload_db_file: Whether to upload the database file.
+        include_db_file: Whether to include the database file.
+        backend_excluded_dirs: A tuple of files or directories to exclude from the backend zip.  Defaults to ().
     """
     zip_dest_dir = zip_dest_dir or Path.cwd()
     zip_dest_dir = Path(zip_dest_dir)
     files_to_exclude = {
-        constants.ComponentName.FRONTEND.zip(),
-        constants.ComponentName.BACKEND.zip(),
+        Path(constants.ComponentName.FRONTEND.zip()).resolve(),
+        Path(constants.ComponentName.BACKEND.zip()).resolve(),
     }
 
     if frontend:
         _zip(
             component_name=constants.ComponentName.FRONTEND,
             target=zip_dest_dir / constants.ComponentName.FRONTEND.zip(),
-            root_dir=prerequisites.get_web_dir() / constants.Dirs.STATIC,
+            root_directory=prerequisites.get_web_dir() / constants.Dirs.STATIC,
             files_to_exclude=files_to_exclude,
-            exclude_venv_dirs=False,
+            exclude_venv_directories=False,
         )
 
     if backend:
         _zip(
             component_name=constants.ComponentName.BACKEND,
             target=zip_dest_dir / constants.ComponentName.BACKEND.zip(),
-            root_dir=Path.cwd(),
-            dirs_to_exclude={"__pycache__"},
-            files_to_exclude=files_to_exclude,
-            top_level_dirs_to_exclude={"assets"},
-            exclude_venv_dirs=True,
-            upload_db_file=upload_db_file,
+            root_directory=Path.cwd(),
+            directory_names_to_exclude={"__pycache__"},
+            files_to_exclude=files_to_exclude | set(backend_excluded_dirs),
+            exclude_venv_directories=True,
+            include_db_file=include_db_file,
             globs_to_include=[
                 str(Path(constants.Dirs.WEB) / constants.Dirs.BACKEND / "*")
             ],
         )
 
 
-def _duplicate_index_html_to_parent_dir(directory: Path):
+def _duplicate_index_html_to_parent_directory(directory: Path):
     """Duplicate index.html in the child directories to the given directory.
 
     This makes accessing /route and /route/ work in production.
@@ -169,7 +183,7 @@ def _duplicate_index_html_to_parent_dir(directory: Path):
                 else:
                     console.debug(f"Skipping {index_html}, already exists at {target}")
             # Recursively call this function for the child directory.
-            _duplicate_index_html_to_parent_dir(child)
+            _duplicate_index_html_to_parent_directory(child)
 
 
 def build():
@@ -200,7 +214,7 @@ def build():
         },
     )
     processes.show_progress("Creating Production Build", process, checkpoints)
-    _duplicate_index_html_to_parent_dir(wdir / constants.Dirs.STATIC)
+    _duplicate_index_html_to_parent_directory(wdir / constants.Dirs.STATIC)
     path_ops.cp(
         wdir / constants.Dirs.STATIC / constants.ReactRouter.SPA_FALLBACK,
         wdir / constants.Dirs.STATIC / "404.html",
@@ -247,6 +261,6 @@ def setup_frontend_prod(
     build()
 
 
-def _looks_like_venv_dir(dir_to_check: str | Path) -> bool:
-    dir_to_check = Path(dir_to_check)
-    return (dir_to_check / "pyvenv.cfg").exists()
+def _looks_like_venv_directory(directory_to_check: str | Path) -> bool:
+    directory_to_check = Path(directory_to_check)
+    return (directory_to_check / "pyvenv.cfg").exists()

reflex/utils/export.py

@@ -18,6 +18,7 @@ def export(
     deploy_url: str | None = None,
     env: constants.Env = constants.Env.PROD,
     loglevel: constants.LogLevel = console._LOG_LEVEL,
+    backend_excluded_dirs: tuple[Path, ...] = (),
 ):
     """Export the app to a zip file.
 
@@ -31,6 +32,7 @@ def export(
         deploy_url: The deploy URL to use. Defaults to None.
         env: The environment to use. Defaults to constants.Env.PROD.
         loglevel: The log level to use. Defaults to console._LOG_LEVEL.
+        backend_excluded_dirs: A tuple of files or directories to exclude from the backend zip.  Defaults to ().
     """
     config = get_config()
 
@@ -70,7 +72,8 @@ def export(
             frontend=frontend,
             backend=backend,
             zip_dest_dir=zip_dest_dir,
-            upload_db_file=upload_db_file,
+            include_db_file=upload_db_file,
+            backend_excluded_dirs=backend_excluded_dirs,
         )
 
     # Post a telemetry event.