chore: microgen - add noxfile for the microgenerator (#2285)

* chore: removes old proof of concept

* removes old __init__.py

* Adds two utility files to handle basic tasks

* Adds a configuration file for the microgenerator

* Removes unused comment

* chore: adds noxfile.py for the microgenerator

Author: chalmerlowe

scripts/microgenerator/noxfile.py

@@ -0,0 +1,382 @@
+# Copyright 2016 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+from functools import wraps
+import pathlib
+import os
+import shutil
+import nox
+import time
+
+
+MYPY_VERSION = "mypy==1.6.1"
+PYTYPE_VERSION = "pytype==2024.9.13"
+BLACK_VERSION = "black==23.7.0"
+BLACK_PATHS = (".",)
+
+DEFAULT_PYTHON_VERSION = "3.9"
+SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.11", "3.12", "3.13"]
+UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.11", "3.12", "3.13"]
+CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+
+
+def _calculate_duration(func):
+    """This decorator prints the execution time for the decorated function."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.monotonic()
+        result = func(*args, **kwargs)
+        end = time.monotonic()
+        total_seconds = round(end - start)
+        hours = total_seconds // 3600  # Integer division to get hours
+        remaining_seconds = total_seconds % 3600  # Modulo to find remaining seconds
+        minutes = remaining_seconds // 60
+        seconds = remaining_seconds % 60
+        human_time = f"{hours:}:{minutes:0>2}:{seconds:0>2}"
+        print(f"Session ran in {total_seconds} seconds ({human_time})")
+        return result
+
+    return wrapper
+
+
+# 'docfx' is excluded since it only needs to run in 'docs-presubmit'
+nox.options.sessions = [
+    "unit",
+    "system",
+    "cover",
+    "lint",
+    "lint_setup_py",
+    "blacken",
+    "mypy",
+    "pytype",
+    "docs",
+]
+
+
+def default(session, install_extras=True):
+    """Default unit test session.
+
+    This is intended to be run **without** an interpreter set, so
+    that the current ``python`` (on the ``PATH``) or the version of
+    Python corresponding to the ``nox`` binary the ``PATH`` can
+    run the tests.
+    """
+
+    constraints_path = str(
+        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    )
+
+    # Install all test dependencies, then install local packages in-place.
+    session.install(
+        "pytest",
+        "google-cloud-testutils",
+        "pytest-cov",
+        "pytest-xdist",
+        "freezegun",
+        "-c",
+        constraints_path,
+    )
+    # We have logic in the magics.py file that checks for whether 'bigquery_magics'
+    # is imported OR not. If yes, we use a context object from that library.
+    # If no, we use our own context object from magics.py. In order to exercise
+    # that logic (and the associated tests) we avoid installing the [ipython] extra
+    # which has a downstream effect of then avoiding installing bigquery_magics.
+    if install_extras and session.python == UNIT_TEST_PYTHON_VERSIONS[0]:
+        install_target = ".[bqstorage,pandas,ipywidgets,geopandas,matplotlib,tqdm,opentelemetry,bigquery_v2]"
+    elif install_extras:  # run against all other UNIT_TEST_PYTHON_VERSIONS
+        install_target = ".[all]"
+    else:
+        install_target = "."
+    session.install("-e", install_target, "-c", constraints_path)
+
+    # Test with some broken "extras" in case the user didn't install the extra
+    # directly. For example, pandas-gbq is recommended for pandas features, but
+    # we want to test that we fallback to the previous behavior. For context,
+    # see internal document go/pandas-gbq-and-bigframes-redundancy.
+    if session.python == UNIT_TEST_PYTHON_VERSIONS[0]:
+        session.run("python", "-m", "pip", "uninstall", "pandas-gbq", "-y")
+
+    session.run("python", "-m", "pip", "freeze")
+
+    # Run py.test against the unit tests.
+    session.run(
+        "py.test",
+        "-n=8",
+        "--quiet",
+        "-W default::PendingDeprecationWarning",
+        "--cov=google/cloud/bigquery",
+        "--cov=tests/unit",
+        "--cov-append",
+        "--cov-config=.coveragerc",
+        "--cov-report=",
+        "--cov-fail-under=0",
+        os.path.join("tests", "unit"),
+        *session.posargs,
+    )
+
+
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
+@_calculate_duration
+def unit(session):
+    """Run the unit test suite."""
+
+    default(session)
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
+def mypy(session):
+    """Run type checks with mypy."""
+
+    session.install("-e", ".[all]")
+    session.install(MYPY_VERSION)
+
+    # Just install the dependencies' type info directly, since "mypy --install-types"
+    # might require an additional pass.
+    session.install(
+        "types-protobuf",
+        "types-python-dateutil",
+        "types-requests",
+        "types-setuptools",
+    )
+    session.run("python", "-m", "pip", "freeze")
+    session.run("mypy", "-p", "google", "--show-traceback")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
+def pytype(session):
+    """Run type checks with pytype."""
+    # An indirect dependecy attrs==21.1.0 breaks the check, and installing a less
+    # recent version avoids the error until a possibly better fix is found.
+    # https://github.com/googleapis/python-bigquery/issues/655
+
+    session.install("attrs==20.3.0")
+    session.install("-e", ".[all]")
+    session.install(PYTYPE_VERSION)
+    session.run("python", "-m", "pip", "freeze")
+    # See https://github.com/google/pytype/issues/464
+    session.run("pytype", "-P", ".", "google/cloud/bigquery")
+
+
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
+@_calculate_duration
+def system(session):
+    """Run the system test suite."""
+
+    constraints_path = str(
+        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    )
+
+    # Sanity check: Only run system tests if the environment variable is set.
+    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
+        session.skip("Credentials must be set via environment variable.")
+
+    # Use pre-release gRPC for system tests.
+    # Exclude version 1.49.0rc1 which has a known issue.
+    # See https://github.com/grpc/grpc/pull/30642
+    session.install("--pre", "grpcio!=1.49.0rc1", "-c", constraints_path)
+
+    # Install all test dependencies, then install local packages in place.
+    session.install(
+        "pytest",
+        "psutil",
+        "pytest-xdist",
+        "google-cloud-testutils",
+        "-c",
+        constraints_path,
+    )
+    if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true":
+        # mTLS test requires pyopenssl and latest google-cloud-storage
+        session.install("google-cloud-storage", "pyopenssl")
+    else:
+        session.install("google-cloud-storage", "-c", constraints_path)
+
+    # Data Catalog needed for the column ACL test with a real Policy Tag.
+    session.install("google-cloud-datacatalog", "-c", constraints_path)
+
+    # Resource Manager needed for test with a real Resource Tag.
+    session.install("google-cloud-resource-manager", "-c", constraints_path)
+
+    if session.python in ["3.11", "3.12"]:
+        extras = "[bqstorage,ipywidgets,pandas,tqdm,opentelemetry]"
+    else:
+        extras = "[all]"
+    session.install("-e", f".{extras}", "-c", constraints_path)
+
+    # Test with some broken "extras" in case the user didn't install the extra
+    # directly. For example, pandas-gbq is recommended for pandas features, but
+    # we want to test that we fallback to the previous behavior. For context,
+    # see internal document go/pandas-gbq-and-bigframes-redundancy.
+    if session.python == SYSTEM_TEST_PYTHON_VERSIONS[0]:
+        session.run("python", "-m", "pip", "uninstall", "pandas-gbq", "-y")
+
+    # print versions of all dependencies
+    session.run("python", "-m", "pip", "freeze")
+
+    # Run py.test against the system tests.
+    session.run(
+        "py.test",
+        "-n=auto",
+        "--quiet",
+        "-W default::PendingDeprecationWarning",
+        os.path.join("tests", "system"),
+        *session.posargs,
+    )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
+def cover(session):
+    """Run the final coverage report.
+
+    This outputs the coverage report aggregating coverage from the unit
+    test runs (not system test runs), and then erases coverage data.
+    """
+
+    session.install("coverage", "pytest-cov")
+    session.run("python", "-m", "pip", "freeze")
+    session.run("coverage", "report", "--show-missing", "--fail-under=100")
+    session.run("coverage", "erase")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
+def lint(session):
+    """Run linters.
+
+    Returns a failure if the linters find linting errors or sufficiently
+    serious code quality issues.
+    """
+
+    session.install("flake8", BLACK_VERSION)
+    session.install("-e", ".")
+    session.run("python", "-m", "pip", "freeze")
+    session.run("flake8", os.path.join("google", "cloud", "bigquery"))
+    session.run("flake8", "tests")
+    session.run("flake8", os.path.join("docs", "samples"))
+    session.run("flake8", os.path.join("docs", "snippets.py"))
+    session.run("flake8", "benchmark")
+    session.run("black", "--check", *BLACK_PATHS)
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
+def lint_setup_py(session):
+    """Verify that setup.py is valid (including RST check)."""
+
+    session.install("docutils", "Pygments")
+    session.run("python", "-m", "pip", "freeze")
+    session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+@_calculate_duration
+def blacken(session):
+    """Run black.
+    Format code to uniform standard.
+    """
+
+    session.install(BLACK_VERSION)
+    session.run("python", "-m", "pip", "freeze")
+    session.run("black", *BLACK_PATHS)
+
+
+@nox.session(python="3.10")
+@_calculate_duration
+def docs(session):
+    """Build the docs."""
+
+    session.install(
+        # We need to pin to specific versions of the `sphinxcontrib-*` packages
+        # which still support sphinx 4.x.
+        # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+        # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+        "sphinxcontrib-applehelp==1.0.4",
+        "sphinxcontrib-devhelp==1.0.2",
+        "sphinxcontrib-htmlhelp==2.0.1",
+        "sphinxcontrib-qthelp==1.0.3",
+        "sphinxcontrib-serializinghtml==1.1.5",
+        "sphinx==4.5.0",
+        "alabaster",
+        "recommonmark",
+    )
+    session.install("google-cloud-storage")
+    session.install("-e", ".[all]")
+
+    shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
+    session.run("python", "-m", "pip", "freeze")
+    session.run(
+        "sphinx-build",
+        "-W",  # warnings as errors
+        "-T",  # show full traceback on exception
+        "-N",  # no colors
+        "-b",
+        "html",
+        "-d",
+        os.path.join("docs", "_build", "doctrees", ""),
+        os.path.join("docs", ""),
+        os.path.join("docs", "_build", "html", ""),
+    )
+
+
+@nox.session(python="3.10")
+@_calculate_duration
+def docfx(session):
+    """Build the docfx yaml files for this library."""
+
+    session.install("-e", ".")
+    session.install(
+        # We need to pin to specific versions of the `sphinxcontrib-*` packages
+        # which still support sphinx 4.x.
+        # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+        # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+        "sphinxcontrib-applehelp==1.0.4",
+        "sphinxcontrib-devhelp==1.0.2",
+        "sphinxcontrib-htmlhelp==2.0.1",
+        "sphinxcontrib-qthelp==1.0.3",
+        "sphinxcontrib-serializinghtml==1.1.5",
+        "gcp-sphinx-docfx-yaml",
+        "alabaster",
+        "recommonmark",
+    )
+
+    shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
+    session.run("python", "-m", "pip", "freeze")
+    session.run(
+        "sphinx-build",
+        "-T",  # show full traceback on exception
+        "-N",  # no colors
+        "-D",
+        (
+            "extensions=sphinx.ext.autodoc,"
+            "sphinx.ext.autosummary,"
+            "docfx_yaml.extension,"
+            "sphinx.ext.intersphinx,"
+            "sphinx.ext.coverage,"
+            "sphinx.ext.napoleon,"
+            "sphinx.ext.todo,"
+            "sphinx.ext.viewcode,"
+            "recommonmark"
+        ),
+        "-b",
+        "html",
+        "-d",
+        os.path.join("docs", "_build", "doctrees", ""),
+        os.path.join("docs", ""),
+        os.path.join("docs", "_build", "html", ""),
+    )