Using list of namespaces

Author: aperez-worklytics

README.md

@@ -49,7 +49,6 @@ kinds: []                          # Empty -> iterate all kinds per namespace
 
 # Optional defaults
 kind: "SourceCollectionStateEntity"  # Default for analyze-fields
-namespace: ""                         # Default namespace for analyze-fields
 
 # Cleanup
 ttl_field: "expireAt"

cli.py

@@ -17,18 +17,12 @@
 ProjectOpt = Annotated[Optional[str], typer.Option("--project", help="GCP/Emulator project id")]
 EmulatorHostOpt = Annotated[Optional[str], typer.Option("--emulator-host", help="Emulator host, e.g. localhost:8010")]
 LogLevelOpt = Annotated[Optional[str], typer.Option("--log-level", help="Logging level")]
-NamespacesOpt = Annotated[
-    Optional[List[str]],
-    typer.Option("--namespace", "-n", help="Namespaces to process (omit to process all)")
-]
 KindsOpt = Annotated[
     Optional[List[str]],
-    typer.Option("--kind", "-k", help="Kinds to process (omit to process all in each namespace)")
+    typer.Option("--kind", "-k", help="Kinds to process (omit or empty to process all in each namespace)")
 ]
-SingleNamespaceOpt = Annotated[Optional[str], typer.Option("--namespace", "-n", help="Namespace to query (omit to use all)")]
 SingleKindOpt = Annotated[Optional[str], typer.Option("--kind", "-k", help="Kind to analyze (falls back to config.kind)")]
 
-
 def _load_cfg(
     config_path: Optional[str],
     project: Optional[str],
@@ -44,25 +38,22 @@ def _load_cfg(
         overrides["log_level"] = log_level
     return load_config(config_path, overrides)
 
-
 @app.command("analyze-kinds")
 def cmd_analyze_kinds(
     config: ConfigOpt = None,
     project: ProjectOpt = None,
     emulator_host: EmulatorHostOpt = None,
     log_level: LogLevelOpt = None,
-    namespace: NamespacesOpt = None,
     kind: KindsOpt = None,
     output: Annotated[Optional[str], typer.Option("--output", help="Output CSV file path")] = None,
 ):
     cfg = _load_cfg(config, project, emulator_host, log_level)
 
-    if namespace:
-        cfg.namespaces = list(namespace)
-    if kind:
-        cfg.kinds = list(kind)
-
+    if kind is not None:
+        # Normalise: treat [""] as empty (all kinds)
+        cfg.kinds = [k for k in kind if k]  # drop empty strings
     rows = analyze_kinds(cfg)
+
     if output:
         with open(output, "w", encoding="utf-8") as fh:
             fh.write("namespace,kind,count,size,bytes\n")
@@ -73,11 +64,10 @@ def cmd_analyze_kinds(
     else:
         print_summary_table(rows)
 
-
 @app.command("analyze-fields")
 def cmd_analyze_fields(
     kind: SingleKindOpt = None,
-    namespace: SingleNamespaceOpt = None,
+    namespace: Annotated[Optional[str], typer.Option("--namespace", "-n", help="Namespace to query (omit to use all)")] = None,
     group_by: Annotated[Optional[str], typer.Option("--group-by", help="Group results by this field value (falls back to config.group_by_field)")] = None,
     only_field: Annotated[Optional[List[str]], typer.Option("--only-field", help="Only consider these fields")] = None,
     config: ConfigOpt = None,
@@ -100,7 +90,7 @@ def cmd_analyze_fields(
         kind=target_kind,
         namespace=target_namespace,
         group_by_field=group_by_field,
-        only_fields=list(only_field) if only_field else None,
+        only_fields=[f for f in only_field] if only_field else None,
     )
 
     if output_json:
@@ -110,14 +100,12 @@ def cmd_analyze_fields(
     else:
         print_field_summary(result)
 
-
 @app.command("cleanup")
 def cmd_cleanup(
     config: ConfigOpt = None,
     project: ProjectOpt = None,
     emulator_host: EmulatorHostOpt = None,
     log_level: LogLevelOpt = None,
-    namespace: NamespacesOpt = None,
     kind: KindsOpt = None,
     ttl_field: Annotated[Optional[str], typer.Option("--ttl-field", help="TTL field name (falls back to config.ttl_field)")] = None,
     delete_missing_ttl: Annotated[Optional[bool], typer.Option("--delete-missing-ttl", help="Delete when TTL field is missing (falls back to config.delete_missing_ttl)")] = None,
@@ -126,10 +114,8 @@ def cmd_cleanup(
 ):
     cfg = _load_cfg(config, project, emulator_host, log_level)
 
-    if namespace:
-        cfg.namespaces = list(namespace)
-    if kind:
-        cfg.kinds = list(kind)
+    if kind is not None:
+        cfg.kinds = [k for k in kind if k]
     if ttl_field is not None:
         cfg.ttl_field = ttl_field
     if delete_missing_ttl is not None:
@@ -141,6 +127,5 @@ def cmd_cleanup(
     deleted_sum = sum(totals.values())
     typer.echo(f"Total entities {'to delete' if dry_run else 'deleted'}: {deleted_sum}")
 
-
 if __name__ == "__main__":
     app()

gcd_tools/analyze_kinds.py

@@ -14,10 +14,8 @@
     format_size,
 )
 
-
 logger = logging.getLogger(__name__)
 
-
 def estimate_entity_count_and_size(
     client: datastore.Client, kind: str, namespace: Optional[str]
 ) -> Tuple[int, int]:
@@ -34,18 +32,15 @@ def estimate_entity_count_and_size(
         count += 1
     return count, total_size
 
-
 def analyze_kinds(config: AppConfig) -> List[Dict]:
     client = build_client(config)
 
-    # Determine namespaces: explicit list, or all
-    namespaces = config.namespaces if config.namespaces else list_namespaces(client)
+    # Thanks to config.py normalisation, [] is the only “all” case
+    namespaces = config.namespaces or list_namespaces(client)
 
     results: List[Dict] = []
     for ns in namespaces:
-        # Determine kinds: explicit list, or all in namespace
-        kinds = config.kinds if config.kinds else list_kinds(client, ns)
-
+        kinds = config.kinds or list_kinds(client, ns)
         logger.info("Analyzing namespace=%s, %d kinds", ns or "(default)", len(kinds))
         for kind in kinds:
             count, total_bytes = estimate_entity_count_and_size(client, kind, ns)
@@ -60,10 +55,9 @@ def analyze_kinds(config: AppConfig) -> List[Dict]:
             )
     return results
 
-
 def print_summary_table(rows: List[Dict]) -> None:
     # Plain stdout table for wide compatibility
     print("namespace,kind,count,size,bytes")
     for r in rows:
         ns = r.get("namespace") or ""
-        print(f"{ns},{r['kind']},{r['count']},{r['size']},{r['bytes']}")
+        print(f"{ns},{r['kind']},{r['count']},{r['size']},{r['bytes']}")

gcd_tools/config.py

@@ -65,6 +65,12 @@ def load_config(path: Optional[str] = None, overrides: Optional[Dict] = None) ->
     config.namespaces = _as_list(merged.get("namespaces"))
     config.kinds = _as_list(merged.get("kinds"))
 
+    # 🛠 Normalise: treat [""] as empty
+    if config.namespaces == [""] or config.namespaces is None:
+        config.namespaces = []
+    if config.kinds == [""] or config.kinds is None:
+        config.kinds = []
+
     # Optional defaults used by some commands
     config.kind = merged.get("kind")
     config.namespace = merged.get("namespace")
@@ -81,6 +87,7 @@ def load_config(path: Optional[str] = None, overrides: Optional[Dict] = None) ->
     return config
 
 
+
 def _configure_logging(level: str) -> None:
     level_value = getattr(logging, level.upper(), logging.INFO)
     logging.basicConfig(level=level_value, format="%(asctime)s | %(levelname)s | %(message)s")
@@ -105,14 +112,22 @@ def build_client(config: AppConfig) -> datastore.Client:
 
 
 def list_namespaces(client: datastore.Client) -> List[str]:
-    # Include default namespace as "" first
+    """
+    Return all namespaces in the datastore, including the default ("").
+    Always queries __namespace__ in the root context so it works in emulator/GCP.
+    """
+    # Include default namespace "" first
     namespaces: List[str] = [""]
-    query = client.query(kind="__namespace__")
+
+    # Force namespace=None to query the metadata root
+    query = client.query(kind="__namespace__", namespace=None)
     query.keys_only()
+
     for entity in query.fetch():
         name = entity.key.name or ""
         if name != "":
             namespaces.append(name)
+
     return namespaces