Merge branch 'main' into main

Author: Rashmi-278

.cursor/mcp.json

@@ -0,0 +1,7 @@
+{
+  "mcpServers": {
+    "oso": {
+      "url": "http://127.0.0.1:8000/sse"
+    }
+  }
+}

.cursor/rules/generate-pyoso-sql-query.mdc

@@ -0,0 +1,25 @@
+---
+description: 
+globs: 
+alwaysApply: true
+---
+**Strict Cursor Rules for Querying the Pyoso Data Lake**
+
+1. **Input → `gather_all_entities()`**
+
+   * Pass the *unaltered* natural-language query (NLQ) directly into `gather_all_entities()` **every time**.
+
+2. **Entities → `query_text2sql_agent()`**
+
+   * Feed **only** the exact output from `gather_all_entities()` into `query_text2sql_agent()`.
+   * Do **not** edit, reorder, or add to the entity list.
+
+3. **SQL String → Further Work**
+
+   * The response from `query_text2sql_agent()` is your finalized, properly formatted SQL query.
+   * Use this SQL string *as-is* for any subsequent execution or analysis.
+
+---
+
+**Workflow Summary:**
+NL query **→** (`gather_all_entities()` + `query_text2sql_agent()`) **→** correct SQL out

.cursor/rules/working-with-pyoso-data.mdc

@@ -0,0 +1,45 @@
+---
+description: 
+globs: 
+alwaysApply: true
+---
+#### 0. One-time setup
+
+Always start by defined the pyoso client with the OSO_API_KEY
+
+```python
+from pyoso import Client
+import os
+client = Client(os.getenv("OSO_API_KEY"))   # never hard-code keys
+```
+
+---
+
+#### 1. Generate SQL
+
+Call the `generate_sql` MCP tool and pass in the user's NL query. This will return the proper SQL to use going forward.
+
+Don't ever call the MCP tool in python code, just use it yourself to gather the proper SQL query. Only the end result SQl query should be written into the code.
+
+```python
+sql_query = 'output of generate_sql MCP tool'
+```
+
+---
+
+#### 2. Run the SQL query across the DB
+
+Pass in the sql_query gathered above into thepyoso client defined above with .to_pandas(), which should return a dataframe result of the query across pyoso's data lake.
+
+```python
+df = client.to_pandas(sql_query)
+```
+
+---
+
+#### 3. (Optional) Analysis
+
+Now, based on the user's request, you are free to continue working with the final dataframe and run any additional analysis they might want done on it.
+
+---
+

.dockerignore

@@ -67,10 +67,6 @@ Dockerfile
 *.pyc
 **/__pycache__
 
-# dbt
-**/target/
-**/dbt_packages/
-
 # Cloudquery
 .cq/
 **/.cq

.env.example

@@ -35,6 +35,16 @@ DAGSTER__CLICKHOUSE__HOST=
 DAGSTER__CLICKHOUSE__USER=
 DAGSTER__CLICKHOUSE__PASSWORD=
 
+# MCP + Text2SQL Agent
+AGENT_VECTOR_STORE__TYPE=local
+AGENT_VECTOR_STORE__DIR=/path/to/your/vector/storage/directory
+
+AGENT_LLM__TYPE=google_genai
+AGENT_LLM__GOOGLE_API_KEY=your_google_genai_api_key_here
+
+AGENT_OSO_API_KEY=your_oso_api_key_here
+AGENT_ARIZE_PHOENIX_USE_CLOUD=0
+
 ###################
 # DEPRECATED
 ###################
@@ -52,5 +62,28 @@ CLOUDSQL_INSTANCE_ID=
 CLOUDSQL_DB_NAME=
 CLOUDSQL_DB_PASSWORD=
 CLOUDSQL_DB_USER=
+
 # Solves the issue of not being able to import the metrics service module when running dagster locally
 PYTHONPATH=warehouse/metrics-service
+
+# Setup agent
+AGENT_OSO_API_KEY=
+# If using a local llm, this will attempt to use ollama with llama3.2:3b
+# See warehouse/oso_agent/util/config.py for more options
+AGENT_LLM__TYPE=local
+
+# To use google's gen ai llm uncomment and add your google api key from google ai studio
+#AGENT_LLM__TYPE=google_genai
+#AGENT_LLM__GOOGLE_API_KEY=
+
+AGENT_VECTOR_STORE__TYPE=local
+# If using the local vector store, set this to a local directory for storing the
+# vector store on disk
+AGENT_VECTOR_STORE__DIR=
+
+# For google vertex ai vector search, set the following options
+#AGENT_VECTOR_STORE__TYPE=google_genai
+#AGENT_VECTOR_STORE__GCS_BUCKET=
+#AGENT_VECTOR_STORE__PROJECT_ID=
+#AGENT_VECTOR_STORE__INDEX_ID=
+#AGENT_VECTOR_STORE__ENDPOINT_ID=

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "npm" # See documentation for possible values
+    directories:
+      - "/"
+      - "**/*"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip" # See documentation for possible values
+    directories:
+      - "/"
+      - "**/*"
+    schedule:
+      interval: "weekly"

.github/workflows/ci-default.yml

@@ -13,6 +13,7 @@ env:
   NEXT_PUBLIC_DB_GRAPHQL_URL: ${{ vars.NEXT_PUBLIC_DB_GRAPHQL_URL }}
   HASURA_URL: ${{ vars.HASURA_URL }}
   OSO_API_KEY: "test"
+  OSO_AGENT_URL: "http://localhost:8888/"
   NEXT_PUBLIC_ALGOLIA_APPLICATION_ID: "test"
   NEXT_PUBLIC_ALGOLIA_API_KEY: "test"
   NEXT_PUBLIC_ALGOLIA_INDEX: "test"

.github/workflows/test-deploy-owners.yml

@@ -45,10 +45,10 @@ jobs:
           cd ops/external-prs &&
           pnpm tools common is-repo-admin ${{ github.event.pull_request.user.login }} --output-file $GITHUB_OUTPUT
 
-      - name: Auto-approve PR if conditions are met
-        run: |
-          cd ops/external-prs &&
-          pnpm tools common attempt-auto-approve ${{ github.event.pull_request.number }} 
+#      - name: Auto-approve PR if conditions are met
+#        run: |
+#          cd ops/external-prs &&
+#          pnpm tools common attempt-auto-approve ${{ github.event.pull_request.number }} 
 
 #      - name: Login to google
 #        uses: "google-github-actions/auth@v2"

.gitignore

@@ -57,11 +57,6 @@ node_modules
 # cloudquery
 .cq/
 
-# poetry.lock files outside of the root. This is to maintain the monorepo with
-# poetry.
-*/**/poetry.lock
-target/
-dbt_packages/
 
 # Python
 *.pyc

.lintstagedrc

@@ -12,9 +12,5 @@
         "uv run ruff check --fix --force-exclude",
         "uv run isort",
         "pnpm pyright"
-    ],
-    "warehouse/dbt/**/*.sql": [
-        "uv run sqlfluff fix -f",
-        "uv run sqlfluff lint"
     ]
 }