Use protobuf definition to generate language independent types (#9)

dhuebner · web-flow · commit dd5cc92aa6b9 · 2025-12-17T14:47:03.000+01:00
* Protobuf definition and TS type generation - initial

# Conflicts:
#	package-lock.json
#	packages/langium-ai-tools/package.json

* Align langium and lai versions
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/examples/example-dsl-evaluator/package.json b/packages/examples/example-dsl-evaluator/package.json
@@ -21,7 +21,7 @@
         "chromadb": "^1.9.2",
         "chromadb-default-embed": "^2.13.2",
         "dotenv": "^16.4.5",
-        "langium-ai-tools": "^0.0.2",
+        "langium-ai-tools": "^4.1.2",
         "ollama": "^0.5.9",
         "openai": "^4.67.3"
     },
diff --git a/packages/examples/example-dsl-evaluator/src/eval-langium.ts b/packages/examples/example-dsl-evaluator/src/eval-langium.ts
@@ -144,9 +144,9 @@ function calculateTriangleAreas(data: MergedEvaluatorResultType): number[] {
         data.warnings,
         1.0 - (data.similarity ?? 0), // Semantic Diff
         (data.errors + data.warnings + data.infos + data.hints + data.unassigned) / 5.0, // Total Diagnostics
-        data.response_length ?? 0,
+        data.responseLength ?? 0,
         data.edit_distance,
-        data._runtime ?? 0
+        data.runtime ?? 0
     ];
 
     const n = values.length;
diff --git a/packages/langium-ai-mcp/package.json b/packages/langium-ai-mcp/package.json
@@ -22,7 +22,6 @@
         "watch": "tsc -w",
         "start": "node .",
         "cstart": "node ./dist/mcp-client.js",
-        "prepare": "npm run build",
         "test": "vitest run"
     },
     "author": {
@@ -39,7 +38,8 @@
     "license": "MIT",
     "dependencies": {
         "@modelcontextprotocol/sdk": "^1.17.4",
-        "langium-ai-tools": "0.0.2"
+        "langium-ai-tools": "~4.1.2",
+        "langium": "~4.1.0"
     },
     "volta": {
         "node": "20.10.0",
diff --git a/packages/langium-ai-tools/package.json b/packages/langium-ai-tools/package.json
@@ -29,10 +29,11 @@
     }
   },
   "scripts": {
-    "build": "tsc",
-    "watch": "tsc -w",
+    "build": "npm run generate && tsc && cp src/interface.proto dist/",
+    "generate": "mkdir -p src/gen && protoc --ts_out=force_disable_services,force_optimize_code_size,eslint_disable:./src/gen --proto_path=src src/interface.proto",
+    "watch": "npm run generate && tsc -w",
     "test": "vitest run",
-    "clean": "rimraf ./dist"
+    "clean": "rimraf ./dist ./src/gen"
   },
   "author": {
     "name": "TypeFox",
@@ -47,11 +48,14 @@
   "license": "MIT",
   "dependencies": {
     "langium": "~4.1.0",
-    "levenshtein-edit-distance": "^3.0.1"
+    "levenshtein-edit-distance": "^3.0.1",
+    "@protobuf-ts/runtime": "^2.11.1"
+  },
+  "devDependencies": {
+    "@protobuf-ts/plugin": "^2.11.1"
   },
   "volta": {
     "node": "20.10.0",
     "npm": "10.2.3"
-  },
-  "devDependencies": {}
-}
+  }
+}
diff --git a/packages/langium-ai-tools/src/analyzer/document-analyzer.ts b/packages/langium-ai-tools/src/analyzer/document-analyzer.ts
@@ -12,6 +12,9 @@ import { LangiumServices } from "langium/lsp";
 import { EvaluationContext } from "../evaluator/document-evaluator.js";
 import { EvaluatorResult } from "../evaluator/evaluator.js";
 import { LangiumEvaluator, LangiumEvaluatorResultData } from "../evaluator/langium-evaluator.js";
+import { EvaluatorResultMsg, SyntaxStatistic } from "../gen/interface.js";
+
+
 
 /**
  * Extends LangiumEvaluator and adds analysis capabilities.
@@ -40,14 +43,40 @@ export class LangiumDocumentAnalyzer<T extends LangiumServices> extends LangiumE
         this.analysisOptions = { ...DEFAULT_OPTIONS, ...analysisOptions };
     }
 
-    evaluateDocument(doc: LangiumDocument, ctx: EvaluationContext): EvaluatorResult<LangiumEvaluatorResultData> {
+    /**
+     * Evaluates a Langium document.
+     * Here we return protocol compatible object EvaluatorResultMsg.
+     * 
+     * @param doc Langium document to evaluate
+     * @param ctx Evaluation context
+     * @returns Evaluation result with syntax statistics in metadata
+     */
+    evaluateDocument(doc: LangiumDocument, ctx: EvaluationContext): EvaluatorResult<LangiumEvaluatorResultData> & EvaluatorResultMsg {
         const validationResult = super.evaluateDocument(doc, ctx);
-        if (this.analysisOptions.analysisMode !== AnalysisMode.NO_STATISTIC && validationResult.data.failures === 0) {
+        if (this.analysisOptions.analysisMode !== AnalysisMode.NO_STATISTIC && validationResult.data && validationResult.data.failures === 0) {
             // Add syntax usage statistics only if build was successful
             const statistics = this.collectSyntaxUsageStatistics(doc, this.services.Grammar);
-            validationResult.metadata[LangiumDocumentAnalyzer.METADATA_KEY] = statistics;
+            validationResult.metadata[LangiumDocumentAnalyzer.METADATA_KEY] = {
+                value: {
+                    oneofKind: 'syntaxStatisticValue',
+                    syntaxStatisticValue: statistics
+                }
+            };
         }
-        return validationResult;
+        // make sure we fulfill the EvaluatorResultMsg interface 
+        return {
+            ...validationResult,
+            data: {
+                ...validationResult.data,
+                diagnostics: validationResult.data.diagnostics.map(diagnostic => {
+                    const code = typeof diagnostic.code === 'number' ? String(diagnostic.code) : diagnostic.code;
+                    return {
+                        ...diagnostic,
+                        code
+                    };
+                })
+            }
+        };
     }
 
     collectSyntaxUsageStatistics(doc: LangiumDocument, grammar: Grammar): SyntaxStatistic {
@@ -173,7 +202,11 @@ export class LangiumDocumentAnalyzer<T extends LangiumServices> extends LangiumE
     extractStatisticsFromResult(result: Partial<EvaluatorResult> | undefined): SyntaxStatistic | undefined {
         const metadata = result?.metadata;
         if (metadata && metadata[LangiumDocumentAnalyzer.METADATA_KEY]) {
-            return metadata[LangiumDocumentAnalyzer.METADATA_KEY] as SyntaxStatistic;
+            const value = metadata[LangiumDocumentAnalyzer.METADATA_KEY].value;
+            if (value.oneofKind === 'syntaxStatisticValue') {
+                return value.syntaxStatisticValue;
+            }
+            return undefined;
         }
         return undefined;
     }
@@ -240,42 +273,3 @@ const DEFAULT_OPTIONS: AnalysisOptions = {
     computeDiversity: true
 };
 
-/**
- * Type representing syntax usage statistics.
- */
-export type SyntaxStatistic = {
-    /** Map of rule names to their usage counts */
-    ruleUsage: Record<string, number>;
-
-    /** Percentage of used rules compared to all available rules */
-    coverage: number;
-
-    /** Diversity metrics for rule usage patterns */
-    diversity: {
-
-        /**
-         * Shannon entropy - information diversity measure.
-         * **Range:** 0 to log₂(n) where n = number of rules.
-         * - **Low (0-1):** dominated by few rules
-         * - **Medium (1-3):** moderate diversity  
-         * - **High (>3):** high diversity
-         */
-        entropy: number;
-
-        /**
-         * Gini coefficient - inequality measure. Range: 0 to 1.
-         * - **Low (0-0.3):** equal distribution
-         * - **Medium (0.3-0.7):** moderate inequality
-         * - **High (0.7-1):** high inequality
-         */
-        giniCoefficient: number;
-
-        /**
-         * Simpson's diversity index - probability that two randomly selected items are different. **Range:** 0 to 1.
-         * - **Low (0-0.3):** low diversity
-         * - **Medium (0.3-0.7):** moderate diversity
-         * - **High (0.7-1):** high diversity
-         */
-        simpsonIndex: number;
-    };
-}
diff --git a/packages/langium-ai-tools/src/evaluator/eval-matrix.ts b/packages/langium-ai-tools/src/evaluator/eval-matrix.ts
@@ -107,7 +107,7 @@ export class EvalMatrix {
                             result.name = `${runner.name} - ${testCase.name} - ${evaluator.name}`;
                         }
                         // add runtime there too, so we have access to it
-                        result.data!._runtime = (runnerEndTime.getTime() - runnerStartTime.getTime()) / 1000.0; // in seconds
+                        result.data!.runtime = (runnerEndTime.getTime() - runnerStartTime.getTime()) / 1000.0; // in seconds
 
                         result.metadata = {
                             runner: runner.name,
diff --git a/packages/langium-ai-tools/src/evaluator/evaluator.ts b/packages/langium-ai-tools/src/evaluator/evaluator.ts
@@ -12,7 +12,7 @@ import { readFileSync, existsSync, readdirSync } from 'fs';
 import * as path from 'path';
 
 export type EvaluatorResultData = Record<string, unknown> & {
-    _runtime?: number;
+    runtime?: number;
 };
 
 /**
diff --git a/packages/langium-ai-tools/src/evaluator/langium-evaluator.ts b/packages/langium-ai-tools/src/evaluator/langium-evaluator.ts
@@ -47,7 +47,7 @@ export interface LangiumEvaluatorResultData extends FailureAwarenessData {
     /**
      * Length of the response in chars
      */
-    response_length: number;
+    responseLength: number;
 
     /**
      * Raw diagnostic data, same which is used to compute the other values above
@@ -109,7 +109,7 @@ export class LangiumEvaluator<T extends LangiumServices> extends AbstractDocumen
             infos: 0,
             hints: 0,
             unassigned: 0,
-            response_length: 0,
+            responseLength: 0,
             diagnostics: []
         };
     }
diff --git a/packages/langium-ai-tools/src/gen/.gitignore b/packages/langium-ai-tools/src/gen/.gitignore
@@ -0,0 +1,2 @@
+# Generated protobuf files
+*.ts
diff --git a/packages/langium-ai-tools/src/index.ts b/packages/langium-ai-tools/src/index.ts
@@ -7,3 +7,4 @@
 export * from './evaluator/index.js';
 export * from './analyzer/index.js';
 export * from './splitter/index.js';
+export * from './gen/interface.js';
diff --git a/packages/langium-ai-tools/src/interface.proto b/packages/langium-ai-tools/src/interface.proto
@@ -0,0 +1,139 @@
+syntax = "proto3";
+
+package langium_ai_tools;
+
+// Position in a text document (line and character offsets)
+message Position {
+    // Line position (zero-based)
+    uint32 line = 1;
+    // Character position (zero-based)  
+    uint32 character = 2;
+}
+
+// Text range with start and end positions
+message Range {
+    // Start position
+    Position start = 1;
+    // End position
+    Position end = 2;
+}
+
+// Diagnostic severity levels
+enum DiagnosticSeverity {
+    DIAGNOSTIC_SEVERITY_UNSPECIFIED = 0;
+    DIAGNOSTIC_SEVERITY_ERROR = 1;
+    DIAGNOSTIC_SEVERITY_WARNING = 2;
+    DIAGNOSTIC_SEVERITY_INFORMATION = 3;
+    DIAGNOSTIC_SEVERITY_HINT = 4;
+}
+
+// VS Code diagnostic information
+message Diagnostic {
+    // Range where the diagnostic applies
+    Range range = 1;
+    // Human-readable message
+    string message = 2;
+    // Severity level
+    optional DiagnosticSeverity severity = 3;
+    // Optional diagnostic code
+    optional string code = 4;
+    // Optional source identifier (e.g., "langium")
+    optional string source = 5;
+}
+
+// Langium-specific evaluator result data
+message LangiumEvaluatorResultDataMsg {
+
+    // Base EvaluatorResultData START
+
+    // Optional runtime in milliseconds
+    optional double runtime = 1;
+
+    // Base EvaluatorResultData END
+
+    // Number of validation failures
+    uint32 failures = 2;
+    // Number of errors
+    uint32 errors = 3;
+    // Number of warnings
+    uint32 warnings = 4;
+    // Number of infos
+    uint32 infos = 5;
+    // Number of hints
+    uint32 hints = 6;
+    // Number of unassigned diagnostics
+    uint32 unassigned = 7;
+    // Length of the response in characters
+    uint32 response_length = 8;
+    // Raw diagnostic data
+    repeated Diagnostic diagnostics = 9;
+}
+
+// Diversity metrics for rule usage patterns
+message DiversityMetrics {
+    // Shannon entropy - information diversity measure
+    // Range: 0 to log₂(n) where n = number of rules
+    // Low (0-1): dominated by few rules
+    // Medium (1-3): moderate diversity
+    // High (>3): high diversity
+    double entropy = 1;
+    
+    // Gini coefficient - inequality measure
+    // Range: 0 to 1
+    // Low (0-0.3): equal distribution
+    // Medium (0.3-0.7): moderate inequality
+    // High (0.7-1): high inequality
+    double gini_coefficient = 2;
+    
+    // Simpson's diversity index - probability that two randomly selected items are different
+    // Range: 0 to 1
+    // Low (0-0.3): low diversity
+    // Medium (0.3-0.7): moderate diversity
+    // High (0.7-1): high diversity
+    double simpson_index = 3;
+}
+
+// Syntax usage statistics for grammar analysis
+message SyntaxStatistic {
+    // Map of rule names to their usage counts
+    map<string, uint32> rule_usage = 1;
+    
+    // Percentage of used rules compared to all available rules
+    double coverage = 2;
+    
+    // Diversity metrics for rule usage patterns
+    DiversityMetrics diversity = 3;
+}
+
+// Nested metadata container (workaround for map in oneof restriction)
+message NestedMetadata {
+    map<string, MetadataValue> values = 1;
+}
+
+// Metadata value wrapper to support different types of metadata
+message MetadataValue {
+    oneof value {
+        // String value
+        string string_value = 1;
+        // Number value
+        double number_value = 2;
+        // Boolean value
+        bool bool_value = 3;
+        // Nested metadata (workaround for map limitation in oneof)
+        NestedMetadata nested_metadata_value = 4;
+        // Syntax statistics value
+        SyntaxStatistic syntax_statistic_value = 5;
+        // Can add more types as needed in the future
+    }
+}
+
+// Generic evaluator result container
+message EvaluatorResultMsg {
+    // Name of this evaluation
+    string name = 1;
+    // Optional metadata as key-value pairs supporting different value types
+    map<string, MetadataValue> metadata = 2;
+    // Langium-specific data
+    LangiumEvaluatorResultDataMsg data = 3;
+}
+

Original file line number	Diff line number	Diff line change
`@@ -107,7 +107,7 @@ export class EvalMatrix {`
`107`	`107`	result.name = `${runner.name} - ${testCase.name} - ${evaluator.name}`;
`108`	`108`	`}`
`109`	`109`	`// add runtime there too, so we have access to it`
`110`		`- result.data!._runtime = (runnerEndTime.getTime() - runnerStartTime.getTime()) / 1000.0; // in seconds`
	`110`	`+ result.data!.runtime = (runnerEndTime.getTime() - runnerStartTime.getTime()) / 1000.0; // in seconds`
`111`	`111`
`112`	`112`	`result.metadata = {`
`113`	`113`	`runner: runner.name,`