From 56479ef4ac00d48fa562445b8d94b75a4503ec6a Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Thu, 5 Dec 2024 13:40:06 -0600
Subject: [PATCH 01/11] bump fireci deps

---
 ci/fireci/setup.cfg | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/ci/fireci/setup.cfg b/ci/fireci/setup.cfg
index 466898d3cb6..6fcdcb19435 100644
--- a/ci/fireci/setup.cfg
+++ b/ci/fireci/setup.cfg
@@ -4,17 +4,17 @@ version = 0.1
 
 [options]
 install_requires =
-        protobuf==3.19
-        click==8.1.3
-        google-cloud-storage==2.5.0
-        mypy==0.991
-        numpy==1.23.1
-        pandas==1.5.1
-        PyGithub==1.55
-        pystache==0.6.0
-        requests==2.23.0
-        seaborn==0.12.1
-        PyYAML==6.0.0
+    protobuf==3.20.3
+    click==8.1.7
+    google-cloud-storage==2.18.2
+    mypy==1.6.0
+    numpy==1.24.4
+    pandas==1.5.3
+    PyGithub==1.58.2
+    pystache==0.6.0
+    requests==2.31.0
+    seaborn==0.12.2
+    PyYAML==6.0.1
 
 [options.extras_require]
 test =

From d93ac5b741f5bbf67b988c4c5313d39c0fedf0dd Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Thu, 5 Dec 2024 13:40:37 -0600
Subject: [PATCH 02/11] Add debug flag to fireci

---
 ci/fireci/fireci/internal.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/ci/fireci/fireci/internal.py b/ci/fireci/fireci/internal.py
index 0950d770fc2..7078528c512 100644
--- a/ci/fireci/fireci/internal.py
+++ b/ci/fireci/fireci/internal.py
@@ -58,6 +58,11 @@ class _CommonOptions:
 
 
 @click.group()
+@click.option(
+  '--debug/--no-debug',
+  help='Set the min loglevel to debug.',
+  default=False
+)
 @click.option(
     '--artifact-target-dir',
     default='_artifacts',
@@ -83,7 +88,7 @@ def main(options, **kwargs):
     setattr(options, k, v)
 
 
-def ci_command(name=None, cls=click.Command, group=main):
+def ci_command(name=None, cls=click.Command, group=main, epilog=None):
   """Decorator to use for CI commands.
 
        The differences from the standard @click.command are:
@@ -94,15 +99,19 @@ def ci_command(name=None, cls=click.Command, group=main):
        :param name:  Optional name of the task. Defaults to the function name that is decorated with this decorator.
        :param cls:   Specifies whether the func is a command or a command group. Defaults to `click.Command`.
        :param group: Specifies the group the command belongs to. Defaults to the `main` command group.
+       :param epilog: Specifies epilog text to show at the end of the help text.
     """
 
   def ci_command(f):
     actual_name = f.__name__ if name is None else name
 
-    @click.command(name=actual_name, cls=cls, help=f.__doc__)
+    @click.command(name=actual_name, cls=cls, help=f.__doc__, epilog=epilog)
     @_pass_options
     @click.pass_context
     def new_func(ctx, options, *args, **kwargs):
+      if options.debug:
+        logging.getLogger('fireci').setLevel(logging.DEBUG)
+      
       with _artifact_handler(
           options.artifact_target_dir,
           options.artifact_patterns,

From 04ab616af9810931cf0e52f165da33f2dd40c121 Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Thu, 5 Dec 2024 13:40:49 -0600
Subject: [PATCH 03/11] Default to info with CI override

---
 ci/fireci/fireci/main.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/ci/fireci/fireci/main.py b/ci/fireci/fireci/main.py
index 9348f69b02c..957cca3c3c8 100644
--- a/ci/fireci/fireci/main.py
+++ b/ci/fireci/fireci/main.py
@@ -20,14 +20,17 @@
 from .internal import main
 
 # Unnecessary on CI as GitHub Actions provides them already.
-asctime_place_holder = '' if os.getenv('CI') else '%(asctime)s '
+is_ci = os.getenv('CI')
+asctime_place_holder = '' if is_ci else '%(asctime)s '
 log_format = f'[%(levelname).1s] {asctime_place_holder}%(name)s: %(message)s'
 logging.basicConfig(
     datefmt='%Y-%m-%d %H:%M:%S %z %Z',
     format=log_format,
     level=logging.INFO,
 )
-logging.getLogger('fireci').setLevel(logging.DEBUG)
+
+level = logging.DEBUG if is_ci else logging.INFO
+logging.getLogger('fireci').setLevel(level)
 
 plugins.discover()
 

From 3733ca0e90569915452b6f0958c4ed83d9bf9585 Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Thu, 5 Dec 2024 13:41:12 -0600
Subject: [PATCH 04/11] Implement clean command

---
 ci/fireci/fireci/ci_utils.py     |   6 ++
 ci/fireci/fireci/dir_utils.py    |  28 ++++++++
 ci/fireci/fireciplugins/clean.py | 108 +++++++++++++++++++++++++++++++
 3 files changed, 142 insertions(+)
 create mode 100644 ci/fireci/fireciplugins/clean.py

diff --git a/ci/fireci/fireci/ci_utils.py b/ci/fireci/fireci/ci_utils.py
index 12ac98b93f6..7e07937d314 100644
--- a/ci/fireci/fireci/ci_utils.py
+++ b/ci/fireci/fireci/ci_utils.py
@@ -61,3 +61,9 @@ def gcloud_identity_token():
   """Returns an identity token with the current gcloud service account."""
   result = subprocess.run(['gcloud', 'auth', 'print-identity-token'], stdout=subprocess.PIPE, check=True)
   return result.stdout.decode('utf-8').strip()
+
+def get_projects(file_path: str = "subprojects.cfg") -> list[str]:
+  """Parses the specified file for a list of projects in the repo."""
+  with open(file_path, 'r') as file:
+    stripped_lines = [line.strip() for line in file]
+    return [line for line in stripped_lines if line and not line.startswith('#')]
diff --git a/ci/fireci/fireci/dir_utils.py b/ci/fireci/fireci/dir_utils.py
index c5aea659d06..89f5415755f 100644
--- a/ci/fireci/fireci/dir_utils.py
+++ b/ci/fireci/fireci/dir_utils.py
@@ -15,6 +15,9 @@
 import contextlib
 import logging
 import os
+import pathlib
+import shutil
+import glob
 
 _logger = logging.getLogger('fireci.dir_utils')
 
@@ -30,3 +33,28 @@ def chdir(directory):
   finally:
     _logger.debug(f'Restoring directory to: {original_dir} ...')
     os.chdir(original_dir)
+
+def rmdir(path: str) -> bool:
+  """Recursively deletes a directory, and returns a boolean indicating if the dir was deleted."""
+  dir = pathlib.Path(path)
+  if not dir.exists():
+    _logger.debug(f"Directory already deleted: {dir}")
+    return False
+
+  _logger.debug(f"Deleting directory: {dir}")
+  shutil.rmtree(dir)
+  return True
+
+def rmglob(pattern: str) -> int:
+  """Deletes all files that match a given pattern, and returns the amount of (root) files deleted"""
+  files = 0
+  for file in glob.glob(os.path.expanduser(pattern)):
+    path = pathlib.Path(file)
+    if path.is_dir():
+      rmdir(path)
+    else:
+      _logger.debug(f"Deleting file: {path}")
+      os.remove(path)
+    files += 1
+
+  return files
diff --git a/ci/fireci/fireciplugins/clean.py b/ci/fireci/fireciplugins/clean.py
new file mode 100644
index 00000000000..c8af6f021ca
--- /dev/null
+++ b/ci/fireci/fireciplugins/clean.py
@@ -0,0 +1,108 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+import logging
+
+from fireci import ci_command
+from fireci import ci_utils
+from fireci import dir_utils
+
+log = logging.getLogger('fireci.clean')
+
+@click.argument("projects", 
+  nargs=-1,
+  type=click.Path(), 
+  required=False
+)
+@click.option('--gradle/--no-gradle', default=False, help="Delete the local .gradle caches.")
+@click.option('--build/--no-build', default=True, help="Delete the local build caches.")
+@click.option('--transforms/--no-transforms', default=False, help="Delete the system-wide transforms cache.")
+@click.option('--build-cache/--no-build-cache', default=False, help="Delete the system-wide build cache.")
+
+@click.option('--deep/--no-deep', default=False, help="Delete all of the system-wide files for gradle.")
+@click.option('--cache/--no-cache', default=False, help="Delete all of the system-wide caches for gradle.")
+@ci_command(epilog="""
+  Clean a subset of projects:
+
+  \b
+  $ fireci clean firebase-common
+  $ fireci clean firebase-common firebase-vertexai
+
+  Clean all projects:
+
+  $ fireci clean
+""")
+def clean(projects, gradle, build, transforms, build_cache, deep, cache):
+  """
+  Delete files cached by gradle.
+
+  Alternative to the standard `gradlew clean`, which runs outside the scope of gradle,
+  and provides deeper cache cleaning capabilities.
+  """
+  if not projects:
+    log.debug("No projects specified, so we're defaulting to all projects.")
+    projects = ci_utils.get_projects()
+
+  cache = cache or deep
+  gradle = gradle or cache
+
+  cleaners = []
+
+  if build:
+    cleaners.append(delete_build)
+  if gradle:
+    cleaners.append(delete_build)
+
+  results = [call_and_sum(projects, cleaner) for cleaner in cleaners]
+  delete_count = sum(map(int, results))
+
+  cleaners = []
+
+  if deep:
+    cleaners.append(delete_deep)
+  elif cache:
+    cleaners.append(delete_cache)
+  else:
+    if transforms:
+      cleaners.append(delete_transforms)
+    if build_cache:
+      cleaners.append(delete_build_cache)
+
+  results = [cleaner() for cleaner in cleaners]
+  delete_count += sum(map(int, results))
+
+  log.info(f"Deleted {delete_count} directories/files")
+
+def call_and_sum(variables, func) -> int:
+  results = map(lambda var: func(var), variables)
+  return sum(map(int, results))
+
+def delete_build(dir: str) -> bool:
+  return dir_utils.rmdir(f"{dir}/build")
+
+def delete_gradle(dir: str) -> bool:
+  return dir_utils.rmdir(f"{dir}/.gradle")
+
+def delete_transforms() -> int:
+  return dir_utils.rmglob("~/.gradle/caches/transforms-*")
+
+def delete_build_cache() -> int:
+  return dir_utils.rmglob("~/.gradle/caches/build-cache-*")
+
+def delete_deep() -> bool:
+  return dir_utils.rmdir("~/.gradle")
+
+def delete_cache() -> bool:
+  return dir_utils.rmdir("~/.gradle/caches")

From b4f6504e3612fa39fadb2e1aca920198b52f64d2 Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Thu, 5 Dec 2024 13:41:27 -0600
Subject: [PATCH 05/11] Add extra useful info to ci readme

---
 ci/README.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/ci/README.md b/ci/README.md
index d1546d1ddbb..3cf10446304 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -22,3 +22,26 @@ This directory contains tooling used to run Continuous Integration tasks.
   ```
   fireci --help
   ```
+
+## Uninstall
+
+If you run into any issues and need to re-install, or uninstall the package, you can do so
+by uninstalling the `fireci` package.
+
+```shell
+pip3 uninstall fireci -y
+```
+
+## Debug
+
+By default, if you're not running `fireci` within the context of CI, the minimum log level is set
+to `INFO`.
+
+To manually set the level to `DEBUG`, you can use the `--debug` flag.
+
+```shell
+fireci --debug clean
+```
+
+> ![NOTE]
+> The `--debug` flag must come _before_ the command.

From 15a5489e56734d24b28adf6bc71d567b3224744f Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Thu, 5 Dec 2024 13:50:20 -0600
Subject: [PATCH 06/11] Update dir_utils.py

---
 ci/fireci/fireci/dir_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/fireci/fireci/dir_utils.py b/ci/fireci/fireci/dir_utils.py
index 89f5415755f..a65c245c6e6 100644
--- a/ci/fireci/fireci/dir_utils.py
+++ b/ci/fireci/fireci/dir_utils.py
@@ -51,7 +51,7 @@ def rmglob(pattern: str) -> int:
   for file in glob.glob(os.path.expanduser(pattern)):
     path = pathlib.Path(file)
     if path.is_dir():
-      rmdir(path)
+      rmdir(file)
     else:
       _logger.debug(f"Deleting file: {path}")
       os.remove(path)

From 2e122e41975839fc1580d37160eec816f16202d9 Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Thu, 5 Dec 2024 14:00:26 -0600
Subject: [PATCH 07/11] Update ci_utils.py

---
 ci/fireci/fireci/ci_utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/fireci/fireci/ci_utils.py b/ci/fireci/fireci/ci_utils.py
index 7e07937d314..07a8eb81e19 100644
--- a/ci/fireci/fireci/ci_utils.py
+++ b/ci/fireci/fireci/ci_utils.py
@@ -16,6 +16,8 @@
 import os
 import subprocess
 
+from typing import List
+
 _logger = logging.getLogger('fireci.ci_utils')
 
 
@@ -62,7 +64,7 @@ def gcloud_identity_token():
   result = subprocess.run(['gcloud', 'auth', 'print-identity-token'], stdout=subprocess.PIPE, check=True)
   return result.stdout.decode('utf-8').strip()
 
-def get_projects(file_path: str = "subprojects.cfg") -> list[str]:
+def get_projects(file_path: str = "subprojects.cfg") -> List[str]:
   """Parses the specified file for a list of projects in the repo."""
   with open(file_path, 'r') as file:
     stripped_lines = [line.strip() for line in file]

From 3e97408d9f923de78aa5d922b37e9f13db2b75de Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Fri, 6 Dec 2024 12:28:11 -0600
Subject: [PATCH 08/11] Address comments

---
 ci/fireci/fireci/ci_utils.py     | 21 ++++++++++++++++++++-
 ci/fireci/fireci/dir_utils.py    |  9 ++++-----
 ci/fireci/fireciplugins/clean.py | 22 ++++++++++++++++------
 ci/fireci/setup.cfg              |  1 +
 4 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/ci/fireci/fireci/ci_utils.py b/ci/fireci/fireci/ci_utils.py
index 07a8eb81e19..1b9e0477e75 100644
--- a/ci/fireci/fireci/ci_utils.py
+++ b/ci/fireci/fireci/ci_utils.py
@@ -16,7 +16,7 @@
 import os
 import subprocess
 
-from typing import List
+from typing import List, Tuple, Union
 
 _logger = logging.getLogger('fireci.ci_utils')
 
@@ -69,3 +69,22 @@ def get_projects(file_path: str = "subprojects.cfg") -> List[str]:
   with open(file_path, 'r') as file:
     stripped_lines = [line.strip() for line in file]
     return [line for line in stripped_lines if line and not line.startswith('#')]
+
+def counts(arr: List[Union[bool, int]]) -> Tuple[int, int]:
+  """Given an array of booleans and ints, returns a tuple mapping of [true, false]. 
+  Positive int values add to the `true` count while values less than one add to `false`.
+  """
+  true_count = 0
+  false_count = 0
+  for value in arr:
+    if isinstance(value, bool):
+      if value:
+        true_count += 1
+      else:
+        false_count += 1
+    elif value >= 1:
+      true_count += value
+    else:
+      false_count += abs(value) if value < 0 else 1
+
+  return true_count, false_count
diff --git a/ci/fireci/fireci/dir_utils.py b/ci/fireci/fireci/dir_utils.py
index a65c245c6e6..bb0dbb0fb44 100644
--- a/ci/fireci/fireci/dir_utils.py
+++ b/ci/fireci/fireci/dir_utils.py
@@ -47,14 +47,13 @@ def rmdir(path: str) -> bool:
 
 def rmglob(pattern: str) -> int:
   """Deletes all files that match a given pattern, and returns the amount of (root) files deleted"""
-  files = 0
-  for file in glob.glob(os.path.expanduser(pattern)):
+  files = glob.glob(os.path.expanduser(pattern))
+  for file in files:
     path = pathlib.Path(file)
     if path.is_dir():
       rmdir(file)
     else:
       _logger.debug(f"Deleting file: {path}")
       os.remove(path)
-    files += 1
-
-  return files
+  
+  return len(files)
diff --git a/ci/fireci/fireciplugins/clean.py b/ci/fireci/fireciplugins/clean.py
index c8af6f021ca..9083857dc57 100644
--- a/ci/fireci/fireciplugins/clean.py
+++ b/ci/fireci/fireciplugins/clean.py
@@ -18,6 +18,8 @@
 from fireci import ci_command
 from fireci import ci_utils
 from fireci import dir_utils
+from typing import Tuple
+from termcolor import colored
 
 log = logging.getLogger('fireci.clean')
 
@@ -63,10 +65,10 @@ def clean(projects, gradle, build, transforms, build_cache, deep, cache):
   if build:
     cleaners.append(delete_build)
   if gradle:
-    cleaners.append(delete_build)
+    cleaners.append(delete_gradle)
 
   results = [call_and_sum(projects, cleaner) for cleaner in cleaners]
-  delete_count = sum(map(int, results))
+  local_count = tuple(map(sum, zip(*results)))
 
   cleaners = []
 
@@ -81,13 +83,21 @@ def clean(projects, gradle, build, transforms, build_cache, deep, cache):
       cleaners.append(delete_build_cache)
 
   results = [cleaner() for cleaner in cleaners]
-  delete_count += sum(map(int, results))
+  system_count = ci_utils.counts(results)
+
+  [deleted, skipped] = tuple(a + b for a, b in zip(local_count, system_count))
+
+  log.info(f"""
+  Clean results:
+
+    {colored("Deleted:", None, attrs=["bold"])} {colored(deleted, "red")}
+    {colored("Already deleted:", None, attrs=["bold"])} {colored(skipped, "grey")}
+  """)
 
-  log.info(f"Deleted {delete_count} directories/files")
 
-def call_and_sum(variables, func) -> int:
+def call_and_sum(variables, func) -> Tuple[int, int]:
   results = map(lambda var: func(var), variables)
-  return sum(map(int, results))
+  return ci_utils.counts(results)
 
 def delete_build(dir: str) -> bool:
   return dir_utils.rmdir(f"{dir}/build")
diff --git a/ci/fireci/setup.cfg b/ci/fireci/setup.cfg
index 6fcdcb19435..4f5909cf2a6 100644
--- a/ci/fireci/setup.cfg
+++ b/ci/fireci/setup.cfg
@@ -15,6 +15,7 @@ install_requires =
     requests==2.31.0
     seaborn==0.12.2
     PyYAML==6.0.1
+    termcolor==2.5.0
 
 [options.extras_require]
 test =

From d366ab61e817c49a62396ca0a655774db27c21cf Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Fri, 6 Dec 2024 12:28:14 -0600
Subject: [PATCH 09/11] Update README.md

---
 ci/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/README.md b/ci/README.md
index 3cf10446304..a4b4eb4799b 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -4,7 +4,7 @@ This directory contains tooling used to run Continuous Integration tasks.
 
 ## Prerequisites
 
-- Requires python3.5+ and setuptools to be installed.
+- Requires python3.9+ and setuptools to be installed.
 
 ## Setup
 

From 8cd31e0a7857f396749985794cfd5beb74dbb771 Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Fri, 6 Dec 2024 12:34:31 -0600
Subject: [PATCH 10/11] Update clean.py

---
 ci/fireci/fireciplugins/clean.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/fireci/fireciplugins/clean.py b/ci/fireci/fireciplugins/clean.py
index 9083857dc57..9f2cd6af9a5 100644
--- a/ci/fireci/fireciplugins/clean.py
+++ b/ci/fireci/fireciplugins/clean.py
@@ -18,7 +18,7 @@
 from fireci import ci_command
 from fireci import ci_utils
 from fireci import dir_utils
-from typing import Tuple
+from typing import Tuple, List, Callable, Union
 from termcolor import colored
 
 log = logging.getLogger('fireci.clean')
@@ -95,8 +95,8 @@ def clean(projects, gradle, build, transforms, build_cache, deep, cache):
   """)
 
 
-def call_and_sum(variables, func) -> Tuple[int, int]:
-  results = map(lambda var: func(var), variables)
+def call_and_sum(variables: List[str], func: Callable[[str], Union[bool, int]]) -> Tuple[int, int]:
+  results = list(map(lambda var: func(var), variables))
   return ci_utils.counts(results)
 
 def delete_build(dir: str) -> bool:

From 7d9c431bfc960df027bf41807e80d5b489c24631 Mon Sep 17 00:00:00 2001
From: Daymon <daymxn@google.com>
Date: Fri, 6 Dec 2024 13:02:26 -0600
Subject: [PATCH 11/11] Update setup.cfg

---
 ci/fireci/setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/fireci/setup.cfg b/ci/fireci/setup.cfg
index 4f5909cf2a6..7b49519871c 100644
--- a/ci/fireci/setup.cfg
+++ b/ci/fireci/setup.cfg
@@ -15,7 +15,7 @@ install_requires =
     requests==2.31.0
     seaborn==0.12.2
     PyYAML==6.0.1
-    termcolor==2.5.0
+    termcolor==2.4.0
 
 [options.extras_require]
 test =