[native_doc_dartifier] Adding RAG Option Using ObjectBox DB

.github/workflows/native_doc_dartifier.yaml

@@ -31,6 +31,10 @@ jobs:
       - uses: subosito/flutter-action@e938fdf56512cc96ef2f93601a5a40bde3801046
         with:
           channel: ${{ matrix.sdk }}
+
+      - name: Download ObjectBox DB .so file
+        run: bash <(curl -s https://raw.githubusercontent.com/objectbox/objectbox-dart/main/install.sh)
+
       - id: install
         name: Install dependencies
         run: flutter pub get

pkgs/native_doc_dartifier/example/dartify_rag_example.dart

@@ -0,0 +1,51 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:native_doc_dartifier/src/context.dart';
+import 'package:native_doc_dartifier/src/dartify_code.dart';
+import 'package:native_doc_dartifier/src/populate_rag.dart';
+
+void main() async {
+  const code = '''public void onClick() {
+    ImageCapture.OutputFileOptions outputFileOptions =
+            new ImageCapture.OutputFileOptions.Builder(new File("...")).build();
+    imageCapture.takePicture(outputFileOptions, cameraExecutor,
+        new ImageCapture.OnImageSavedCallback() {
+            @Override
+            public void onImageSaved(ImageCapture.OutputFileResults outputFileResults) {
+                // insert your code here.
+            }
+            @Override
+            public void onError(ImageCaptureException error) {
+                // insert your code here.
+            }
+       }
+    );
+}
+''';
+
+  final bindingsFile = File('example/camerax.dart');
+
+  final bindingsPath = bindingsFile.absolute.path;
+
+  // Populate RAG with relevant information from the bindings file
+  // This step is crucial and must be done only once before using RAG
+  await populateRAG(Directory.current.path, bindingsPath);
+
+  try {
+    final context = await Context.create(
+      Directory.current.path,
+      bindingsPath,
+      // Indicate that we are using RAG
+      usingRag: true,
+    );
+    final dartCode = await dartifyNativeCode(code, context);
+    print(dartCode);
+  } catch (e) {
+    stderr.writeln('Error: $e');
+    exit(1);
+  }
+}

pkgs/native_doc_dartifier/lib/native_doc_dartifier.dart

@@ -3,3 +3,4 @@
 // BSD-style license that can be found in the LICENSE file.
 
 // TODO: Export any libraries intended for clients of this package.
+export 'src/rag.dart';

pkgs/native_doc_dartifier/lib/src/ast.dart

@@ -174,6 +174,26 @@ class PackageSummary {
     }
     return buffer.toString();
   }
+
+  List<String> getRAGSummaries() {
+    final summaries = <String>[];
+    final topLevelDeclerations = StringBuffer();
+
+    if (packageName.isNotEmpty) {
+      topLevelDeclerations.writeln('// From: $packageName');
+    }
+    for (final function in topLevelFunctions) {
+      topLevelDeclerations.writeln('$function;');
+    }
+    for (final variable in topLevelVariables) {
+      topLevelDeclerations.writeln('$variable;');
+    }
+
+    summaries.add(topLevelDeclerations.toString());
+    summaries.addAll(classesSummaries.map((c) => c.toDartLikeRepresentation()));
+
+    return summaries;
+  }
 }
 
 class LibraryClassSummary {

pkgs/native_doc_dartifier/lib/src/context.dart

@@ -14,8 +14,10 @@ import 'package:path/path.dart' as p;
 
 import 'ast.dart';
 import 'public_abstractor.dart';
+import 'rag.dart';
 
 class Context {
+  final RAG? rag;
   final String projectAbsolutePath;
   final String bindingsFileAbsolutePath;
   final List<String> importedPackages = [];
@@ -25,15 +27,18 @@ class Context {
   Context._({
     required this.projectAbsolutePath,
     required this.bindingsFileAbsolutePath,
-  });
+    required bool usingRag,
+  }) : rag = (usingRag ? RAG.instance : null);
 
   static Future<Context> create(
     String projectAbsolutePath,
-    String bindingsFileAbsolutePath,
-  ) async {
+    String bindingsFileAbsolutePath, {
+    bool usingRag = false,
+  }) async {
     final context = Context._(
       projectAbsolutePath: projectAbsolutePath,
       bindingsFileAbsolutePath: bindingsFileAbsolutePath,
+      usingRag: usingRag,
     );
     await context._init();
     return context;
@@ -46,15 +51,17 @@ class Context {
       exit(1);
     }
 
-    // Get the bindings file summary
-    final abstractor = PublicAbstractor();
-    parseString(
-      content: await bindingsFile.readAsString(),
-    ).unit.visitChildren(abstractor);
-    bindingsSummary.addAll(abstractor.getBindingsClassesSummary());
+    if (rag == null) {
+      // Get the bindings file summary
+      final abstractor = PublicAbstractor();
+      parseString(
+        content: await bindingsFile.readAsString(),
+      ).unit.visitChildren(abstractor);
+      bindingsSummary.addAll(abstractor.getBindingsClassesSummary());
 
-    // Get the packages classes summary, that are imported in the bindings file
-    await _getLibrariesSummary(projectAbsolutePath, bindingsFileAbsolutePath);
+      // Get packages classes summary, that are imported in the bindings file
+      await _getLibrariesSummary(projectAbsolutePath, bindingsFileAbsolutePath);
+    }
   }
 
   Future<void> _getLibrariesSummary(
@@ -155,14 +162,24 @@ class Context {
     return;
   }
 
-  String toDartLikeRepresentation() {
+  /// It will return the full context,
+  /// If [rag] is null or the given [querySnippet] is empty.
+  Future<String> toDartLikeRepresentation(String querySnippet) async {
     final buffer = StringBuffer();
-    for (final classSummary in bindingsSummary) {
-      buffer.writeln(classSummary.toDartLikeRepresentation());
-    }
-    for (final packageSummary in packageSummaries) {
-      buffer.writeln(packageSummary.toDartLikeRepresentation());
+    if (rag != null && querySnippet.isNotEmpty) {
+      final documents = await rag!.queryRAG(querySnippet);
+      for (final classSummary in documents) {
+        buffer.writeln(classSummary);
+      }
+    } else {
+      for (final classSummary in bindingsSummary) {
+        buffer.writeln(classSummary.toDartLikeRepresentation());
+      }
+      for (final packageSummary in packageSummaries) {
+        buffer.writeln(packageSummary.toDartLikeRepresentation());
+      }
     }
+
     final dartLikeRepresentation = buffer.toString().replaceAll('jni\$_.', '');
     return dartLikeRepresentation;
   }

pkgs/native_doc_dartifier/lib/src/dartify_code.dart

@@ -32,7 +32,7 @@ Future<String> dartifyNativeCode(String sourceCode, Context context) async {
 
   final translatePrompt = TranslatePrompt(
     sourceCode,
-    context.toDartLikeRepresentation(),
+    await context.toDartLikeRepresentation(sourceCode),
   );
 
   final chatSession = model.startChat();

pkgs/native_doc_dartifier/lib/src/populate_rag.dart

@@ -0,0 +1,33 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'context.dart';
+import 'rag.dart';
+
+Future<void> populateRAG(
+  String projectAbsolutePath,
+  String bindingsFileAbsolutePath,
+) async {
+  final context = await Context.create(
+    projectAbsolutePath,
+    bindingsFileAbsolutePath,
+  );
+
+  final listOfSummaries = <String>[];
+
+  final bindingsClassSummary =
+      context.bindingsSummary.map((c) => c.toDartLikeRepresentation()).toList();
+
+  listOfSummaries.addAll(bindingsClassSummary);
+
+  for (final package in context.packageSummaries) {
+    final packageClassesSummary =
+        package.classesSummaries
+            .map((c) => c.toDartLikeRepresentation())
+            .toList();
+    listOfSummaries.addAll(packageClassesSummary);
+  }
+
+  await RAG.instance.addAllDocumentsToRag(listOfSummaries);
+}

pkgs/native_doc_dartifier/lib/src/rag.dart

@@ -0,0 +1,131 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:google_generative_ai/google_generative_ai.dart';
+import 'package:objectbox_dev/objectbox.g.dart';
+import 'package:objectbox_dev/rag_models.dart';
+
+class RAG {
+  static final RAG _instance = RAG._internal();
+  static RAG get instance => _instance;
+  late final Store _store;
+  late final Box<ClassSummaryRAGModel> _classSummaryBox;
+
+  RAG._internal() {
+    _store = openStore();
+    _classSummaryBox = _store.box<ClassSummaryRAGModel>();
+  }
+
+  void close() {
+    _store.close();
+  }
+
+  Future<List<String>> queryRAG(
+    String javaSnippet, {
+    int numRetrievedDocs = 20,
+  }) async {
+    final apiKey = Platform.environment['GEMINI_API_KEY'];
+    if (apiKey == null) {
+      stderr.writeln(r'No $GEMINI_API_KEY environment variable');
+      exit(1);
+    }
+
+    final embeddingModel = GenerativeModel(
+      apiKey: apiKey,
+      model: 'gemini-embedding-001',
+    );
+
+    final queryEmbeddings = await embeddingModel
+        .embedContent(Content.text(javaSnippet))
+        .then((embedContent) => embedContent.embedding.values);
+
+    // The Database makes use of HNSW algorithm for embeddings search which
+    // is O(log n) in search time complexity better than O(n).
+    // but the tradeoff that it gets the approximate nearest neighbors
+    // instead of the exact ones
+    // so make it to return approx 100 nearest neighbors and then get the top K.
+    final query =
+        _classSummaryBox
+            .query(
+              ClassSummaryRAGModel_.embeddings.nearestNeighborsF32(
+                queryEmbeddings,
+                100,
+              ),
+            )
+            .build();
+    query.limit = numRetrievedDocs;
+    final resultWithScore = query.findWithScores();
+    print('RAG query returned ${resultWithScore.length} results.');
+    final result = resultWithScore.map((e) => e.object.summary).toList();
+
+    query.close();
+    return result;
+  }
+
+  Future<void> addAllDocumentsToRag(List<String> classesSummary) async {
+    final apiKey = Platform.environment['GEMINI_API_KEY'];
+    if (apiKey == null) {
+      stderr.writeln(r'No $GEMINI_API_KEY environment variable');
+      exit(1);
+    }
+
+    final embeddingModel = GenerativeModel(
+      apiKey: apiKey,
+      model: 'gemini-embedding-001',
+    );
+
+    // TODO: Check if the documents already exist in the RAG and skip
+    // adding them instead of clearing all and re-adding them.
+    print('Clearing existing RAG documents...');
+    _classSummaryBox.removeAll();
+
+    const batchSize = 100;
+    final batchEmbededContent = <BatchEmbedContentsResponse>[];
+
+    for (var i = 0; i < classesSummary.length; i += batchSize) {
+      final upperbound =
+          i + batchSize < classesSummary.length
+              ? i + batchSize
+              : classesSummary.length;
+      print('Processing batch from $i to $upperbound...');
+      final batch = classesSummary.sublist(
+        i,
+        i + batchSize > classesSummary.length
+            ? classesSummary.length
+            : i + batchSize,
+      );
+
+      final batchResponse = await embeddingModel.batchEmbedContents(
+        List.generate(
+          batch.length,
+          (index) => EmbedContentRequest(Content.text(batch[index])),
+        ),
+      );
+
+      batchEmbededContent.add(batchResponse);
+
+      // Quota limit is 100 requests per minute
+      await Future<void>.delayed(const Duration(minutes: 1));
+    }
+
+    final embeddings = <List<double>>[];
+    for (final response in batchEmbededContent) {
+      for (final embedContent in response.embeddings) {
+        embeddings.add(embedContent.values);
+      }
+    }
+
+    final classSummaries = <ClassSummaryRAGModel>[];
+    for (var i = 0; i < classesSummary.length; i++) {
+      classSummaries.add(
+        ClassSummaryRAGModel(classesSummary[i], embeddings[i]),
+      );
+    }
+    _classSummaryBox.putMany(classSummaries);
+
+    print('Added ${classesSummary.length} documents to the RAG.');
+  }
+}

pkgs/native_doc_dartifier/objectbox_dev/.gitignore

@@ -0,0 +1,7 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/
+
+# Avoid committing pubspec.lock for library packages; see
+# https://dart.dev/guides/libraries/private-files#pubspeclock.
+pubspec.lock

pkgs/native_doc_dartifier/objectbox_dev/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+
+- Initial version.