Merge pull request #195 from ZAK1504/feature/bigquery-adapter

Add BigQuery adapter implementation (Fixes #117)

Author: shiva-intugle

docsite/docs/connectors/bigquery.md

@@ -0,0 +1,109 @@
+---
+sidebar_position: 7
+---
+
+# BigQuery
+
+`intugle` integrates with Google BigQuery, allowing you to read data from your datasets for profiling, analysis, and data product generation.
+
+## Installation
+
+To use `intugle` with BigQuery, you must install the optional dependencies:
+
+```bash
+pip install "intugle[bigquery]"
+```
+
+This installs the `google-cloud-bigquery` library.
+
+## Configuration
+
+To connect to your BigQuery project, you must provide connection credentials in a `profiles.yml` file at the root of your project. The adapter looks for a top-level `bigquery:` key.
+
+**Example `profiles.yml`:**
+
+```yaml
+bigquery:
+  name: my_bigquery_source
+  project_id: <your_gcp_project_id>
+  dataset: <your_dataset_name>
+  location: US  # Optional, defaults to US
+  credentials_path: /path/to/service-account-credentials.json  # Optional
+```
+
+### Authentication Options
+
+1. **Service Account JSON File** (Recommended for production):
+   - Set `credentials_path` to your service account JSON file.
+   - The service account needs **BigQuery Data Viewer** and **BigQuery Job User** roles.
+
+2. **Application Default Credentials** (For development):
+   - Omit `credentials_path`.
+   - Uses `gcloud auth application-default login`.
+   - Or uses environment variable `GOOGLE_APPLICATION_CREDENTIALS`.
+
+## Usage
+
+### Reading Data from BigQuery
+
+To include a BigQuery table or view in your `SemanticModel`, define it in your input dictionary with `type: "bigquery"` and use the `identifier` key to specify the table name.
+
+:::caution Important
+The dictionary key for your dataset (e.g., `"my_table"`) must exactly match the table name specified in the `identifier`.
+:::
+
+```python
+from intugle import SemanticModel
+
+datasets = {
+    "my_table": {
+        "identifier": "my_table", # Must match the key above
+        "type": "bigquery"
+    },
+    "another_view": {
+        "identifier": "another_view",
+        "type": "bigquery"
+    }
+}
+
+# Initialize the semantic model
+sm = SemanticModel(datasets, domain="Analytics")
+
+# Build the model as usual
+sm.build()
+```
+
+### Materializing Data Products
+
+When you use the `DataProduct` class with a BigQuery connection, the resulting data product can be materialized as a new **table** or **view** directly within your target dataset.
+
+:::caution
+**Beta Feature:** The DataProduct feature for BigQuery is currently in beta. If you encounter any issues, please raise them on our [GitHub issues page](https://github.com/Intugle/data-tools/issues).
+:::
+
+```python
+from intugle import DataProduct
+
+etl_model = {
+    "name": "top_users",
+    "fields": [
+        {"id": "users.id", "name": "user_id"},
+        {"id": "users.name", "name": "user_name"},
+    ]
+}
+
+dp = DataProduct()
+
+# Materialize as a view (default)
+dp.build(etl_model, materialize="view")
+
+# Materialize as a table
+dp.build(etl_model, materialize="table")
+```
+
+:::info Required Permissions
+To successfully materialise data products, the Service Account or User must have the following privileges:
+*   `roles/bigquery.dataViewer` - Read table data
+*   `roles/bigquery.jobUser` - Run queries
+*   `roles/bigquery.dataEditor` - Create tables and views
+:::

docsite/docs/connectors/implementing-a-connector.md

@@ -1,5 +1,5 @@
 ---
-sidebar_position: 7
+sidebar_position: 8
 ---
 
 # Implementing a Connector

pyproject.toml

@@ -69,6 +69,10 @@ postgres = [
     "asyncpg>=0.30.0",
     "sqlglot>=27.20.0",
 ]
+bigquery = [
+    "google-cloud-bigquery>=3.11.0",
+    "sqlglot>=27.20.0",
+]
 sqlserver = [
     "mssql-python>=0.13.1",
     "sqlglot>=27.20.0",

src/intugle/adapters/factory.py

@@ -36,6 +36,7 @@ def is_safe_plugin_name(plugin_name: str) -> bool:
     "intugle.adapters.types.mariadb.mariadb",
     "intugle.adapters.types.sqlserver.sqlserver",
     "intugle.adapters.types.sqlite.sqlite",
+    "intugle.adapters.types.bigquery.bigquery",
     "intugle.adapters.types.oracle.oracle",
 ]
 

src/intugle/adapters/models.py

@@ -19,6 +19,7 @@ def get_dataset_data_type() -> type:
 if TYPE_CHECKING:
     import pandas as pd
 
+    from intugle.adapters.types.bigquery.models import BigQueryConfig
     from intugle.adapters.types.databricks.models import DatabricksConfig
     from intugle.adapters.types.duckdb.models import DuckdbConfig
     from intugle.adapters.types.mariadb.models import MariaDBConfig
@@ -28,7 +29,7 @@ def get_dataset_data_type() -> type:
     from intugle.adapters.types.sqlite.models import SqliteConfig
     from intugle.adapters.types.sqlserver.models import SQLServerConfig
 
-    DataSetData = pd.DataFrame | DuckdbConfig | SnowflakeConfig | DatabricksConfig | PostgresConfig | SQLServerConfig | SqliteConfig | OracleConfig | MariaDBConfig
+    DataSetData = pd.DataFrame | DuckdbConfig | SnowflakeConfig | DatabricksConfig | PostgresConfig | SQLServerConfig | SqliteConfig | OracleConfig | MariaDBConfig | BigQueryConfig
 else:
     # At runtime, this is dynamically determined
     DataSetData = Any

src/intugle/adapters/types/bigquery/__init__.py

@@ -0,0 +1 @@
+# BigQuery adapter for Intugle