Merge remote-tracking branch 'upstream/main' into feature/engine_extensions

Author: LantaoJin

.claude/commands/dedupe.md

@@ -0,0 +1,33 @@
+---
+allowed-tools: Bash(gh:*), Bash(./scripts/comment-on-duplicates.sh:*)
+description: Find duplicate GitHub issues
+---
+
+Find up to 3 likely duplicate issues for a given GitHub issue.
+
+Follow these steps precisely:
+
+1. Use `gh issue view <number>` to read the issue. If the issue is closed, or is broad product feedback without a specific bug/feature request, or already has a duplicate detection comment (containing `<!-- duplicate-detection -->`), stop and report why you are not proceeding.
+
+2. Summarize the issue's core problem in 2-3 sentences. Identify the key terms, error messages, and affected components.
+
+3. Search for potential duplicates using **at least 3 different search strategies**. Run these searches in parallel. **Only consider issues with a lower issue number** (older issues) as potential originals — skip any result with a number >= the current issue. Also skip issues already labeled `duplicate`.
+   - `gh search issues "<exact error message or key phrase>" --repo $GITHUB_REPOSITORY --state open -- -label:duplicate --limit 15 --json number,title | jq '[.[] | select(.number < <current-issue-number>)]'`
+   - `gh search issues "<component or feature keywords>" --repo $GITHUB_REPOSITORY --state open -- -label:duplicate --limit 15 --json number,title | jq '[.[] | select(.number < <current-issue-number>)]'`
+   - `gh search issues "<alternate description of the problem>" --repo $GITHUB_REPOSITORY --state open -- -label:duplicate --limit 15 --json number,title | jq '[.[] | select(.number < <current-issue-number>)]'`
+   - `gh search issues "<key terms>" --repo $GITHUB_REPOSITORY --state all -- -label:duplicate --limit 10 --json number,title | jq '[.[] | select(.number < <current-issue-number>)]'` (include closed issues for reference)
+
+4. For each candidate issue that looks like a potential match, read it with `gh issue view <number>` to verify it is truly about the same problem. Filter out false positives — issues that merely share keywords but describe different problems.
+
+5. If you find 1-3 genuine duplicates, post the result using the comment script:
+   ```
+   ./scripts/comment-on-duplicates.sh --base-issue <issue-number> --potential-duplicates <dup1> [dup2] [dup3]
+   ```
+
+6. If no genuine duplicates are found, report that no duplicates were detected and take no further action.
+
+Important notes:
+- Only flag issues as duplicates when you are confident they describe the **same underlying problem**
+- Prefer open issues as duplicates, but closed issues can be referenced too
+- Do not flag the issue as a duplicate of itself
+- The base issue number is the last part of the issue reference (e.g., for `owner/repo/issues/42`, the number is `42`)

.github/workflows/auto-close-duplicates.yml

@@ -0,0 +1,132 @@
+name: Auto-close duplicate issues
+
+on:
+  schedule:
+    - cron: "0 9 * * *"
+  workflow_dispatch:
+
+permissions:
+  issues: write
+
+jobs:
+  auto-close-duplicates:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Close stale duplicate issues
+        uses: actions/github-script@v7
+        env:
+          GRACE_DAYS: ${{ vars.DUPLICATE_GRACE_DAYS || '7' }}
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const graceDays = parseInt(process.env.GRACE_DAYS, 10) || 7;
+            const GRACE_PERIOD_MS = graceDays * 24 * 60 * 60 * 1000;
+            const now = Date.now();
+
+            // Find all open issues with the duplicate label
+            const issues = await github.paginate(github.rest.issues.listForRepo, {
+              owner,
+              repo,
+              state: 'open',
+              labels: 'duplicate',
+              per_page: 100,
+            });
+
+            console.log(`Found ${issues.length} open issues with duplicate label`);
+
+            let closedCount = 0;
+
+            for (const issue of issues) {
+              console.log(`Processing issue #${issue.number}: ${issue.title}`);
+
+              // Get comments to find the duplicate detection comment
+              const comments = await github.rest.issues.listComments({
+                owner,
+                repo,
+                issue_number: issue.number,
+                per_page: 100,
+              });
+
+              // Find the duplicate detection comment (posted by our script)
+              const dupeComments = comments.data.filter(c =>
+                c.body.includes('<!-- duplicate-detection -->')
+              );
+
+              if (dupeComments.length === 0) {
+                console.log(`  No duplicate detection comment found, skipping`);
+                continue;
+              }
+
+              const lastDupeComment = dupeComments[dupeComments.length - 1];
+              const dupeCommentAge = now - new Date(lastDupeComment.created_at).getTime();
+
+              if (dupeCommentAge < GRACE_PERIOD_MS) {
+                const daysLeft = ((GRACE_PERIOD_MS - dupeCommentAge) / (24 * 60 * 60 * 1000)).toFixed(1);
+                console.log(`  Duplicate comment is too recent (${daysLeft} days remaining), skipping`);
+                continue;
+              }
+
+              // Check for human comments after the duplicate detection comment
+              const humanCommentsAfter = comments.data.filter(c =>
+                new Date(c.created_at) > new Date(lastDupeComment.created_at) &&
+                c.user.type !== 'Bot' &&
+                !c.body.includes('<!-- duplicate-detection -->') &&
+                !c.body.includes('automatically closed as a duplicate')
+              );
+
+              if (humanCommentsAfter.length > 0) {
+                console.log(`  Has ${humanCommentsAfter.length} human comment(s) after detection, skipping`);
+                continue;
+              }
+
+              // Check for thumbs-down reaction from the issue author
+              const reactions = await github.rest.reactions.listForIssueComment({
+                owner,
+                repo,
+                comment_id: lastDupeComment.id,
+                per_page: 100,
+              });
+
+              const authorThumbsDown = reactions.data.some(r =>
+                r.user.id === issue.user.id && r.content === '-1'
+              );
+
+              if (authorThumbsDown) {
+                console.log(`  Issue author gave thumbs-down on duplicate comment, skipping`);
+                continue;
+              }
+
+              // Extract the primary duplicate issue number from the comment
+              const dupeMatch = lastDupeComment.body.match(/#(\d+)/);
+              const dupeNumber = dupeMatch ? dupeMatch[1] : 'unknown';
+
+              // Close the issue
+              console.log(`  Closing as duplicate of #${dupeNumber}`);
+
+              await github.rest.issues.update({
+                owner,
+                repo,
+                issue_number: issue.number,
+                state: 'closed',
+                state_reason: 'duplicate',
+              });
+
+              await github.rest.issues.addLabels({
+                owner,
+                repo,
+                issue_number: issue.number,
+                labels: ['autoclose'],
+              });
+
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number: issue.number,
+                body: `This issue has been automatically closed as a duplicate of #${dupeNumber}.\n\nIf this is incorrect, please reopen this issue or create a new one.\n\n🤖 Generated with [Claude Code](https://claude.ai/code)`,
+              });
+
+              closedCount++;
+            }
+
+            console.log(`Done. Closed ${closedCount} duplicate issue(s).`);

.github/workflows/claude-dedupe-issues.yml

@@ -0,0 +1,43 @@
+name: Claude Issue Dedupe
+
+on:
+  issues:
+    types: [opened]
+  workflow_dispatch:
+    inputs:
+      issue_number:
+        description: 'Issue number to check for duplicates'
+        required: true
+        type: string
+
+permissions:
+  contents: read
+  issues: write
+  id-token: write
+
+jobs:
+  dedupe:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.BEDROCK_ACCESS_ROLE }}
+          aws-region: us-east-1
+
+      - name: Run duplicate detection
+        uses: anthropics/claude-code-action@v1
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          DUPLICATE_GRACE_DAYS: ${{ vars.DUPLICATE_GRACE_DAYS }}
+        with:
+          use_bedrock: "true"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          allowed_bots: "github-actions[bot]"
+          prompt: "/dedupe ${{ github.repository }}/issues/${{ github.event.issue.number || inputs.issue_number }}"
+          claude_args: "--model us.anthropic.claude-sonnet-4-5-20250929-v1:0"

.github/workflows/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+  - package-ecosystem: "gradle"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "08:00"
+      timezone: "America/Los_Angeles"
+    labels:
+      - "skip-changelog"
+    group:
+      all-dependencies:
+        patterns:
+          - "*"

.github/workflows/remove-duplicate-on-activity.yml

@@ -0,0 +1,42 @@
+name: Remove duplicate label on activity
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  issues: write
+
+jobs:
+  remove-duplicate:
+    if: |
+      github.event.issue.state == 'open' &&
+      contains(github.event.issue.labels.*.name, 'duplicate') &&
+      github.event.comment.user.type != 'Bot'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove duplicate label
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const issueNumber = context.issue.number;
+            const commenter = context.payload.comment.user.login;
+
+            console.log(`Removing duplicate label from issue #${issueNumber} due to comment from ${commenter}`);
+
+            try {
+              await github.rest.issues.removeLabel({
+                owner,
+                repo,
+                issue_number: issueNumber,
+                name: 'duplicate',
+              });
+              console.log(`Successfully removed duplicate label from issue #${issueNumber}`);
+            } catch (error) {
+              if (error.status === 404) {
+                console.log(`duplicate label was already removed from issue #${issueNumber}`);
+              } else {
+                throw error;
+              }
+            }

.whitesource

@@ -11,5 +11,11 @@
   },
   "issueSettings": {
     "minSeverityLevel": "LOW"
+  },
+  "remediateSettings": {
+    "addLabels": ["skip-changelog"],
+    "workflowRules": {
+      "enabled": true
+    }
   }
-}
+}

api/README.md

@@ -8,7 +8,8 @@ This module provides components organized into two main areas aligned with the [
 
 ### Unified Language Specification
 
-- **`UnifiedQueryPlanner`**: Accepts PPL (Piped Processing Language) or SQL queries and returns Calcite `RelNode` logical plans as intermediate representation.
+- **`UnifiedQueryParser`**: Parses PPL (Piped Processing Language) or SQL queries and returns the native parse result (`UnresolvedPlan` for PPL, `SqlNode` for Calcite SQL).
+- **`UnifiedQueryPlanner`**: Accepts PPL or SQL queries and returns Calcite `RelNode` logical plans as intermediate representation.
 - **`UnifiedQueryTranspiler`**: Converts Calcite logical plans (`RelNode`) into SQL strings for various target databases using different SQL dialects.
 
 ### Unified Execution Runtime
@@ -42,6 +43,20 @@ UnifiedQueryContext context = UnifiedQueryContext.builder()
     .build();
 ```
 
+### UnifiedQueryParser
+
+Use `UnifiedQueryParser` to parse queries into their native parse tree. The parser is owned by `UnifiedQueryContext` and returns the native parse result for each language.
+
+```java
+// PPL parsing
+UnresolvedPlan ast = (UnresolvedPlan) context.getParser().parse("source = logs | where status = 200");
+
+// SQL parsing (with QueryType.SQL context)
+SqlNode sqlNode = (SqlNode) sqlContext.getParser().parse("SELECT * FROM logs WHERE status = 200");
+```
+
+Callers can then use each language's native visitor infrastructure (`AbstractNodeVisitor` for PPL, `SqlBasicVisitor` for Calcite SQL) on the typed result for further analysis.
+
 ### UnifiedQueryPlanner
 
 Use `UnifiedQueryPlanner` to parse and analyze PPL or SQL queries into Calcite logical plans. The planner accepts a `UnifiedQueryContext` and can be reused for multiple queries.
@@ -179,6 +194,59 @@ try (UnifiedQueryContext context = UnifiedQueryContext.builder()
 }
 ```
 
+## Profiling
+
+The unified query API supports the same [profiling capability](../docs/user/ppl/interfaces/endpoint.md#profile-experimental) as the PPL REST endpoint. When enabled, each unified query component automatically collects per-phase timing metrics. For code outside unified query components (e.g., `PreparedStatement.executeQuery()` or response formatting), `context.measure()` records custom phases into the same profile.
+
+```java
+try (UnifiedQueryContext context = UnifiedQueryContext.builder()
+    .language(QueryType.PPL)
+    .catalog("catalog", schema)
+    .defaultNamespace("catalog")
+    .profiling(true)
+    .build()) {
+
+  // Auto-profiled: ANALYZE
+  RelNode plan = new UnifiedQueryPlanner(context).plan(query);
+
+  // Auto-profiled: OPTIMIZE
+  PreparedStatement stmt = new UnifiedQueryCompiler(context).compile(plan);
+
+  // User-profiled via measure()
+  ResultSet rs = context.measure(MetricName.EXECUTE, stmt::executeQuery);
+  String json = context.measure(MetricName.FORMAT, () -> formatter.format(result));
+
+  // Retrieve profile snapshot
+  QueryProfile profile = context.getProfile();
+}
+```
+
+The returned `QueryProfile` follows the same JSON structure as the REST API:
+
+```json
+{
+  "summary": {
+    "total_time_ms": 33.34
+  },
+  "phases": {
+    "analyze": { "time_ms": 8.68 },
+    "optimize": { "time_ms": 18.2 },
+    "execute": { "time_ms": 4.87 },
+    "format": { "time_ms": 0.05 }
+  },
+  "plan": {
+    "node": "EnumerableCalc",
+    "time_ms": 4.82,
+    "rows": 2,
+    "children": [
+      { "node": "CalciteEnumerableIndexScan", "time_ms": 4.12, "rows": 2 }
+    ]
+  }
+}
+```
+
+When profiling is disabled (the default), all components execute with zero overhead.
+
 ## Development & Testing
 
 A set of unit tests is provided to validate planner behavior.