draft proof of concept microgenerator 2

Author: chalmerlowe

scripts/microgenerator/bigqueryclient_config.yaml

@@ -0,0 +1,21 @@
+# TODO: add header if needed.
+
+include_class_name_patterns:
+  - Client
+
+exclude_class_name_patterns: []
+
+include_method_name_patterns:
+  - batch_delete_
+  - cancel_
+  - create_
+  - delete_
+  - get_
+  - insert_
+  - list_
+  - patch_
+  - undelete_
+  - update_
+
+exclude_method_name_patterns:
+  - get_mtls_endpoint_and_cert_source

scripts/microgenerator/bigqueryclient_generator.py

@@ -0,0 +1,169 @@
+import ast
+import os
+from collections import defaultdict
+
+import jinja2
+
+from config_helper import (
+    CLASSES_TO_INCLUDE,
+    # CLASSES_TO_EXCLUDE, # Not currently being used.
+    METHODS_TO_INCLUDE,
+    METHODS_TO_EXCLUDE,
+)
+
+# Constants
+BASE_DIR = "google/cloud/bigquery_v2/services"
+FILES_TO_PARSE = [
+    os.path.join(root, file)
+    for root, _, files in os.walk(BASE_DIR)
+    for file in files
+    if file.endswith(".py")
+]
+
+
+def create_tree(file_path):
+    with open(file_path, "r") as source:
+        tree = ast.parse(source.read())
+    return tree
+
+
+def _extract_classes(tree):
+    """Extracts class nodes from an AST."""
+    classes = []
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef) and node.name.endswith(
+            *CLASSES_TO_INCLUDE
+        ):  # TODO: currently this is one class. Refactor if necessary
+            classes.append(node)
+    return classes
+
+
+def _extract_methods(class_node):
+    """Extracts method nodes from a class node."""
+    return (m for m in class_node.body if isinstance(m, ast.FunctionDef))
+
+
+def _process_method(method, class_name, parsed_data):
+    """Processes a single method and updates parsed_data."""
+    method_name = method.name
+    if any(method_name.startswith(prefix) for prefix in METHODS_TO_INCLUDE) and not any(
+        method_name.startswith(prefix) for prefix in METHODS_TO_EXCLUDE
+    ):
+        parameters = [arg.arg for arg in method.args.args + method.args.kwonlyargs]
+        parsed_data[class_name][method_name] = parameters
+
+
+def parse_files(file_paths):
+    """
+    Parse a list of Python files and extract information about classes,
+    methods, and parameters.
+
+    Args:
+        file_paths (list): List of file paths to parse.
+
+    Returns:
+        Defaultdict with zero or more entries.
+    """
+
+    parsed_data = defaultdict(dict)
+
+    for file_path in file_paths:
+        tree = create_tree(file_path)
+
+        for class_ in _extract_classes(tree):
+            class_name = class_.name
+            parsed_data[class_name]
+
+            for method in _extract_methods(class_):
+                _process_method(method, class_name, parsed_data)
+
+    return parsed_data
+
+
+def _format_args(method_args):
+    """Formats method arguments for use in creating a method definition
+    and a method call.
+    """
+    args_for_def = ", ".join(method_args)
+    args_for_call = ", ".join([f"{arg}={arg}" for arg in method_args if arg != "self"])
+    return args_for_def, args_for_call
+
+
+def _format_class_name(method_name, suffix="Request"):
+    """Formats a class name from a method name.
+
+    Example:
+        list_datasets -> ListDatasetsRequest
+    """
+    return "".join(word.capitalize() for word in method_name.split("_")) + suffix
+
+
+def generate_client_class_source(data):
+    """
+    Generates the BigQueryClient source code using a Jinja2 template.
+
+    Args:
+        data: A dictionary where keys are *ServiceClient class names and
+              values are dictionaries of methods for that client.
+
+    Returns:
+        A string containing the complete, formatted Python source code
+        for the BigQueryClient class.
+    """
+
+    # TODO: move template strings to a separate file.
+    class_template_string = """\
+class BigQueryClient:
+    def __init__(self):
+        self._clients = {}
+
+{% for method in methods %}
+    def {{ method.name }}({{ method.args_for_def }}):
+        \"\"\"A generated method to call the BigQuery API.\"\"\"
+
+        if "{{ method.class_name }}" not in self._clients:
+            from google.cloud.bigquery_v2 import {{ method.class_name }}
+            self._clients["{{ method.class_name }}"] = {{ method.class_name }}()
+
+        client = self._clients["{{ method.class_name }}"]
+        from google.cloud.bigquery_v2 import types
+        request = types.{{ method.request_class_name }}({{ method.args_for_call }})
+        return client.{{ method.name }}(request=request)
+
+{% endfor %}
+"""
+
+    # Prepare the context for the template.
+    # We transform the input data into a flat list of methods
+    methods_context = []
+    for class_name, methods in data.items():
+        for method_name, method_args in methods.items():
+            args_for_def, args_for_call = _format_args(method_args)
+            request_class_name = _format_class_name(method_name)
+            methods_context.append(
+                {
+                    "name": method_name,
+                    "class_name": class_name,
+                    "args_for_def": args_for_def,
+                    "args_for_call": args_for_call,
+                    "request_class_name": request_class_name,
+                }
+            )
+
+    # Create a Jinja2 Template object and render it with the context.
+    template = jinja2.Template(class_template_string, trim_blocks=True)
+    generated_code = template.render(methods=methods_context)
+
+    return generated_code
+
+
+if __name__ == "__main__":
+    data = parse_files(FILES_TO_PARSE)
+
+    final_code = generate_client_class_source(data)
+    # TODO: write final code to file.
+
+    print(final_code)
+
+    # Ensure black gets called on the generated source files as a final step.

scripts/microgenerator/config_helper.py

@@ -0,0 +1,39 @@
+# config_helper.py
+
+import yaml
+import os
+
+
+def load_config_yaml(filepath):
+    """Loads configuration from a YAML file."""
+    try:
+        with open(filepath, "r") as f:
+            config = yaml.safe_load(f)
+        return config
+    except FileNotFoundError:
+        print(f"Error: Configuration file '{filepath}' not found.")
+        return None
+    except yaml.YAMLError as e:
+        print(f"Error: Could not load YAML from '{filepath}': {e}")
+        return None
+
+
+# Determine the absolute path to the config file relative to this file.
+# This makes the path robust to where the script is run from.
+_CONFIG_FILE_PATH = os.path.join(
+    os.path.dirname(__file__), "bigqueryclient_config.yaml"
+)
+
+config_data = load_config_yaml(_CONFIG_FILE_PATH)
+
+if config_data:
+    CLASSES_TO_INCLUDE = config_data.get("include_class_name_patterns", [])
+    CLASSES_TO_EXCLUDE = config_data.get("exclude_class_name_patterns", [])
+    METHODS_TO_INCLUDE = config_data.get("include_method_name_patterns", [])
+    METHODS_TO_EXCLUDE = config_data.get("exclude_method_name_patterns", [])
+else:
+    # Define default empty values if the config fails to load
+    CLASSES_TO_INCLUDE = []
+    CLASSES_TO_EXCLUDE = []
+    METHODS_TO_INCLUDE = []
+    METHODS_TO_EXCLUDE = []