chore: Script to inspect and clean up stale GCFs (#331)

Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
- [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery-dataframes/issues/new/choose) before writing your code!  That way we can discuss the change, evaluate designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [ ] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)

Fixes internal issue 319307783 🦕

### Usage:
```bash
$ python scripts/manage_cloud_functions.py --help
usage: manage_cloud_functions.py [-h] -p PROJECT_ID [-r REGIONS] {summary,cleanup} ...

Manage cloud functions created to serve bigframes remote functions.

options:
  -h, --help            show this help message and exit
  -p PROJECT_ID, --project-id PROJECT_ID
                        GCP project-id.
  -r REGIONS, --regions REGIONS
                        Cloud functions region(s). If multiple regions, Specify comma separated (e.g. region1,region2)

subcommands:
  {summary,cleanup}
    summary             BigFrames cloud functions summary.
    cleanup             BigFrames cloud functions clean up.


$ python scripts/manage_cloud_functions.py summary --help
usage: manage_cloud_functions.py summary [-h]

Show the bigframes cloud functions summary.

options:
  -h, --help  show this help message and exit


$ python scripts/manage_cloud_functions.py cleanup --help
usage: manage_cloud_functions.py cleanup [-h] [-n NUMBER]

Delete the stale bigframes cloud functions.

options:
  -h, --help            show this help message and exit
  -n NUMBER, --number NUMBER
                        Number of stale (more than a day old) cloud functions to clean up.
(venv) shobs@shobs-ct-3:~/code/bigframes1$ 

```

### Example:
```bash
$ python scripts/manage_cloud_functions.py -p bigframes-dev summary
us-central1: Total=1412, Recent=86, OlderThanADay=1326
europe-west4: Total=270, Recent=24, OlderThanADay=246
southamerica-west1: Total=269, Recent=23, OlderThanADay=246
europe-west1: Total=262, Recent=23, OlderThanADay=239
asia-southeast1: Total=260, Recent=18, OlderThanADay=242
us-east1: Total=1, Recent=0, OlderThanADay=1

$ python scripts/manage_cloud_functions.py -p bigframes-dev -r us-central1,europe-west4 summary
us-central1: Total=1412, Recent=85, OlderThanADay=1327
europe-west4: Total=270, Recent=24, OlderThanADay=246

$ python scripts/manage_cloud_functions.py -p bigframes-dev -r us-central1,europe-west4 cleanup -n 2
[us-central1]: deleted [1] projects/bigframes-dev/locations/us-central1/functions/bigframes-597cc02ef5ce0525e4f51697b5a83b6c-3pfpu6gu last updated on 2024-01-08 21:47:58.503628+00:00
[us-central1]: deleted [2] projects/bigframes-dev/locations/us-central1/functions/bigframes-68f796a13666bb3bfe354dd1adaeef71 last updated on 2024-01-09 21:52:49.620259+00:00
[europe-west4]: deleted [1] projects/bigframes-dev/locations/europe-west4/functions/bigframes-558d0ca6649537a9e45896faf08b0a7a last updated on 2024-01-12 21:15:04.379198+00:00
[europe-west4]: deleted [2] projects/bigframes-dev/locations/europe-west4/functions/bigframes-4b7705561ec336ed80722a8e6e56ac41 last updated on 2024-01-08 05:34:59.331828+00:00

$ python scripts/manage_cloud_functions.py -p bigframes-dev -r us-central1,europe-west4 summary
us-central1: Total=1410, Recent=85, OlderThanADay=1325
europe-west4: Total=269, Recent=25, OlderThanADay=244
```

Author: shobsi

scripts/manage_cloud_functions.py

@@ -0,0 +1,195 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from datetime import datetime
+import sys
+import time
+
+import google.api_core.exceptions
+from google.cloud import functions_v2
+
+GCF_REGIONS_ALL = [
+    "asia-east1",
+    "asia-east2",
+    "asia-northeast1",
+    "asia-northeast2",
+    "europe-north1",
+    "europe-southwest1",
+    "europe-west1",
+    "europe-west2",
+    "europe-west4",
+    "europe-west8",
+    "europe-west9",
+    "us-central1",
+    "us-east1",
+    "us-east4",
+    "us-east5",
+    "us-south1",
+    "us-west1",
+    "asia-east2",
+    "asia-northeast3",
+    "asia-southeast1",
+    "asia-southeast2",
+    "asia-south1",
+    "asia-south2",
+    "australia-southeast1",
+    "australia-southeast2",
+    "europe-central2",
+    "europe-west2",
+    "europe-west3",
+    "europe-west6",
+    "northamerica-northeast1",
+    "northamerica-northeast2",
+    "southamerica-east1",
+    "southamerica-west1",
+    "us-west2",
+    "us-west3",
+    "us-west4",
+]
+
+GCF_CLIENT = functions_v2.FunctionServiceClient()
+
+
+def get_bigframes_functions(project, region):
+    parent = f"projects/{args.project_id}/locations/{region}"
+    functions = GCF_CLIENT.list_functions(
+        functions_v2.ListFunctionsRequest(parent=parent)
+    )
+    # Filter bigframes created functions
+    functions = [
+        function
+        for function in functions
+        if function.name.startswith(
+            f"projects/{args.project_id}/locations/{region}/functions/bigframes-"
+        )
+    ]
+
+    return functions
+
+
+def summarize_gcfs(args):
+    """Summarize number of bigframes cloud functions in various regions."""
+
+    region_counts = {}
+    for region in args.regions:
+        functions = get_bigframes_functions(args.project_id, region)
+        functions_count = len(functions)
+
+        # Exclude reporting regions with 0 bigframes GCFs
+        if functions_count == 0:
+            continue
+
+        # Count how many GCFs are newer than a day
+        recent = 0
+        for f in functions:
+            age = datetime.now() - datetime.fromtimestamp(f.update_time.timestamp())
+            if age.days <= 0:
+                recent += 1
+
+        region_counts[region] = (functions_count, recent)
+
+    for item in sorted(
+        region_counts.items(), key=lambda item: item[1][0], reverse=True
+    ):
+        region = item[0]
+        count, recent = item[1]
+        print(
+            "{}: Total={}, Recent={}, OlderThanADay={}".format(
+                region, count, recent, count - recent
+            )
+        )
+
+
+def cleanup_gcfs(args):
+    """Clean-up bigframes cloud functions in the given regions."""
+    max_delete_per_region = args.number
+
+    for region in args.regions:
+        functions = get_bigframes_functions(args.project_id, region)
+        count = 0
+        for f in functions:
+            age = datetime.now() - datetime.fromtimestamp(f.update_time.timestamp())
+            if age.days > 0:
+                try:
+                    count += 1
+                    GCF_CLIENT.delete_function(name=f.name)
+                    print(
+                        f"[{region}]: deleted [{count}] {f.name} last updated on {f.update_time}"
+                    )
+                    if count >= max_delete_per_region:
+                        break
+                    # Mostly there is a 60 mutations per minute quota, we want to use 10% of
+                    # that for this clean-up, i.e. 6 mutations per minute. So wait for
+                    # 60/6 = 10 seconds
+                    time.sleep(10)
+                except google.api_core.exceptions.ResourceExhausted:
+                    # Stop deleting in this region for now
+                    print(
+                        f"Cannot delete any more functions in region {region} due to quota exhaustion. Please try again later."
+                    )
+                    break
+
+
+def list_str(values):
+    return [val for val in values.split(",") if val]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Manage cloud functions created to serve bigframes remote functions."
+    )
+    parser.add_argument(
+        "-p",
+        "--project-id",
+        type=str,
+        required=True,
+        action="store",
+        help="GCP project-id.",
+    )
+    parser.add_argument(
+        "-r",
+        "--regions",
+        type=list_str,
+        required=False,
+        default=GCF_REGIONS_ALL,
+        action="store",
+        help="Cloud functions region(s). If multiple regions, Specify comma separated (e.g. region1,region2)",
+    )
+
+    subparsers = parser.add_subparsers(title="subcommands", required=True)
+    parser_summary = subparsers.add_parser(
+        "summary",
+        help="BigFrames cloud functions summary.",
+        description="Show the bigframes cloud functions summary.",
+    )
+    parser_summary.set_defaults(func=summarize_gcfs)
+    parser_cleanup = subparsers.add_parser(
+        "cleanup",
+        help="BigFrames cloud functions clean up.",
+        description="Delete the stale bigframes cloud functions.",
+    )
+    parser_cleanup.add_argument(
+        "-n",
+        "--number",
+        type=int,
+        required=False,
+        default=100,
+        action="store",
+        help="Number of stale (more than a day old) cloud functions to clean up.",
+    )
+    parser_cleanup.set_defaults(func=cleanup_gcfs)
+
+    args = parser.parse_args(sys.argv[1:])
+    args.func(args)