[docs-agent] Release of Docs Agent v.0.3.3

What's changed:

- Added a new post-processing feature to delete existing chunks
  in databases if they are no longer found in source dataset.
- Updated the pre-processing module to make sure text chunks
  are not bigger than 5 KB for generating embeddings.
- Added a feature to display the distribution of chunk sizes
  after running `agent chunk`.
- Added more `agent` commandlines and options.
- Updated the "Rewrite" button to the "Feedback" button by default.
- Bug fixes.

Author: kyolee415

examples/gemini/python/docs-agent/README.md

@@ -172,7 +172,13 @@ Authorize Google Cloud credentials on your host machine:
 
 2. Copy the `client_secret.json` file to your host machine.
 
-3. To authenticate credentials, run the following command in the directory of
+3. Install the Google Cloud SDK on your host machine:
+
+   ```
+   sudo apt install google-cloud-sdk
+   ```
+
+4. To authenticate credentials, run the following command in the directory of
    the host machine where the `client_secret.json` file is located:
 
    ```
@@ -181,10 +187,7 @@ Authorize Google Cloud credentials on your host machine:
 
    This command opens a browser and asks to log in using your Google account.
 
-   **Note**: If the `gcloud` command doesn’t exist, install the Google Cloud SDK
-   on your host machine: `sudo apt install google-cloud-sdk`
-
-4. Follow the instructions on the browser and click **Allow** to authenticate.
+5. Follow the instructions on the browser and click **Allow** to authenticate.
 
    This saves the authenticated credentials for Docs Agent
    (`application_default_credentials.json`) in the `$HOME/.config/gcloud/`

examples/gemini/python/docs-agent/apps_script/README.md

@@ -54,6 +54,7 @@ Do the following:
 1. Repeat the steps until all the `.gs` files are copied to your Apps Script project.
 1. Click the **+** icon next to **Services**.
 1. Scroll down and click **Drive API**.
+1. Select **v2**.
 1. Click **Add**.
 
 You are now ready to edit the parameters on the `main.gs` file to select a folder

examples/gemini/python/docs-agent/apps_script/drive_to_markdown.gs

@@ -13,25 +13,55 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+function convertDriveFolderToMDForDocsAgent(folderName, outputFolderName=""){
+  gdoc_count = 0;
+  pdf_count = 0;
+  new_file_count = 0;
+  updated_file_count = 0;
+  unchanged_file_count = 0;
+  gdoc_count, pdf_count, new_file_count, updated_file_count, unchanged_file_count = convertDriveFolder(folderName, outputFolderName=outputFolderName)
+  let conversion_count = pdf_count + gdoc_count
+  let file_count = new_file_count + updated_file_count + unchanged_file_count
+  Logger.log("Converted a total of: " + gdoc_count + " Google Doc files.");
+  Logger.log("Converted a total of: " + pdf_count + " PDF files.");
+  Logger.log("Converted a grand total of: " + conversion_count + " files.");
+  Logger.log("New files: " + new_file_count)
+  Logger.log("Updated a total of: " + updated_file_count + " files.")
+  Logger.log("Files that haven't changed: " + unchanged_file_count);
+  Logger.log("Input directory had a total of: " + file_count + " files.")
+}
+
+function convertDriveFolder(folderName, outputFolderName="", indexFile="") {
 
-function convertDriveFolderToMDForDocsAgent(folderName) {
   //Checks if input folder exists or exits
   if(folderExistsInput(folderName)){
     var file_count = 0;
     var folders = DriveApp.getFoldersByName(folderName);
-    Logger.log("Output directory: "+ folderName + "-output");
-    var folderOutput = folderName + "-output";
-    var output_file_name = folderName + "-index";
+    if (outputFolderName=="") {
+      var folderOutput = folderName + "-output";
+      var output_file_name = folderName + "-index";
+    }
+    else {
+      var folderOutput = outputFolderName + "-output";
+      var output_file_name = outputFolderName + "-index";
+    }
+    Logger.log("Output directory: "+ folderOutput);
     folderExistsOrCreate(folderOutput);
     var folderOutputObj = DriveApp.getFoldersByName(folderOutput);
     if (folderOutputObj.hasNext()){
       var folderOutputName = folderOutputObj.next();
     }
-    var sheet = checkIndexOutputOrCreate(output_file_name, folderOutputName);
-    var timeZone = Session.getScriptTimeZone();
-    var date = Utilities.formatDate(new Date(), timeZone, "MM-dd-yyyy HH:mm:ss z");
-    sheet.appendRow(["Created: ", date])
-    sheet.appendRow(["Name","ID", "URL", "Markdown ID", "Markdown Output", "Date Created", "Last Updated", "Type", "Folder", "MD5 hash", "Status"]);
+    if (indexFile=="") {
+      var sheet = checkIndexOutputOrCreate(output_file_name, folderOutputName);
+      var timeZone = Session.getScriptTimeZone();
+      var date = Utilities.formatDate(new Date(), timeZone, "MM-dd-yyyy HH:mm:ss z");
+      sheet.appendRow(["Created: ", date])
+      sheet.appendRow(["Name","ID", "URL", "Markdown ID", "Markdown Output", "Date Created", "Last Updated", "Type", "Folder", "MD5 hash", "Status"]);
+    }
+    else {
+      var sheet = indexFile
+    }
+    // var sheet_id = sheet.getId();
     var foldersnext = folders.next();
     var myfiles = foldersnext.getFiles();
     var new_file_count = 0;
@@ -54,6 +84,22 @@ function convertDriveFolderToMDForDocsAgent(folderName) {
       else{
         var fid = myfile.getId();
       }
+      if (ftype == "application/vnd.google-apps.folder") {
+        var folder = DriveApp.getFolderById(fid);
+        Logger.log("Sub-directory: " + folder);
+        sub_gdoc_count = 0;
+        sub_pdf_count = 0;
+        sub_new_file_count = 0;
+        sub_updated_file_count = 0;
+        sub_unchanged_file_count = 0;
+        sub_gdoc_count, sub_pdf_count, sub_new_file_count, sub_updated_file_count, sub_unchanged_file_count = convertDriveFolder(folder, outputFolderName=foldersnext, indexFile=sheet);
+        gdoc_count += sub_gdoc_count;
+        pdf_count += sub_pdf_count;
+        new_file_count += sub_new_file_count;
+        updated_file_count += sub_updated_file_count;
+        unchanged_file_count += sub_unchanged_file_count;
+        continue;
+      }
       var fname = sanitizeFileName(myfile.getName());
       var fdate = myfile.getLastUpdated();
       var furl = myfile.getUrl();
@@ -158,7 +204,7 @@ function convertDriveFolderToMDForDocsAgent(folderName) {
           var saved_file_id = saved_file.getId();
           Logger.log("Finished converting file: "+ fname + " to markdown.");
           Logger.log("Markdown file: " + saved_file);
-          Logger.log("Clearing temporary gdoc: " );
+          Logger.log("Clearing temporary gdoc" );
           let output_file = DriveApp.getFileById(output_id);
           output_file.setTrashed(true);
           status = "New content";
@@ -182,19 +228,13 @@ function convertDriveFolderToMDForDocsAgent(folderName) {
         hash_str,
         status,
       ];
-      row_number = file_count + start_data_row;
       sheet.appendRow(metadata);
+      // Return final row to inserRichText into correct rows
+      row_number = sheet.getLastRow();
       insertRichText(sheet, original_chip, "C", row_number);
       insertRichText(sheet, md_chip, "E", row_number);
       insertRichText(sheet, folder_chip, "I", row_number);
     }
   }
-  let conversion_count = pdf_count + gdoc_count
-  Logger.log("Converted a total of: " + gdoc_count + " Google Doc files.");
-  Logger.log("Converted a total of: " + pdf_count + " PDF files.");
-  Logger.log("Converted a grand total of: " + conversion_count + " files.");
-  Logger.log("New files: " + new_file_count)
-  Logger.log("Updated a total of: " + updated_file_count + " files.")
-  Logger.log("Files that haven't changed: " + unchanged_file_count);
-  Logger.log("Input directory had a total of: " + file_count + " files.")
+  return gdoc_count, pdf_count, new_file_count, updated_file_count, unchanged_file_count
 }

examples/gemini/python/docs-agent/apps_script/helper_functions.gs

@@ -67,18 +67,24 @@ function checkFileExists(fileName,folderName){
 
 // Function to check if an index output sheet exists or creates it. Returns the file object
 // Specify the file output name and outputdirectory
-function checkIndexOutputOrCreate(fileName, folderOutput) {
+function checkIndexOutputOrCreate(fileName, folderOutput, indexFileID="") {
   var timeZone = Session.getScriptTimeZone();
   var date = Utilities.formatDate(new Date(), timeZone, "MM-dd-yyyy hh:mm:ss");
   let file = {title: fileName, mimeType: MimeType.GOOGLE_SHEETS, parents: [{id: folderOutput.getId()}]}
   let params = "title='" + fileName + "' and parents in '" + folderOutput.getId() + "'";
   let file_search = DriveApp.searchFiles(params);
   if (file_search.hasNext()) {
-    let fileId = file_search.next().getId();
+    if (indexFileID=="") {
+      var fileId = file_search.next().getId();
+    }
+    else {
+      var fileId = indexFileID;
+    }
     var sheet = SpreadsheetApp.openById(fileId);
     Logger.log("File index: " + fileName + " exists.");
     var sheet_index = sheet.getSheetByName("Index");
-    if (sheet.getSheetByName("Backup")){
+    // Checks to see if this is a sub directory
+    if (sheet.getSheetByName("Backup")) {
       var sheet_backup = sheet.getSheetByName("Backup");
       sheet.deleteSheet(sheet_backup);
     }

examples/gemini/python/docs-agent/docs/cli-reference.md

@@ -26,6 +26,15 @@ by running the `agent chunk` command):
 agent populate
 ```
 
+### Populate a vector database and delete stale text chunks
+
+The command below deletes stale entries in the existing vector database
+before populating it with the new text chunks:
+
+```sh
+agent populate --enable_delete_chunks
+```
+
 ### Show the Docs Agent configuration
 
 The command below prints all the fields and values in the current
@@ -35,6 +44,15 @@ The command below prints all the fields and values in the current
 agent show-config
 ```
 
+### Clean up the Docs Agent development environment
+
+The command below deletes development databases specified in the
+`config.yaml` file:
+
+```sh
+agent cleanup-dev
+```
+
 ## Docs Agent chatbot web app
 
 ### Launch the Docs Agent web app
@@ -53,6 +71,24 @@ The command below launches the Docs Agent web app to run on port 5005:
 agent chatbot --port 5005
 ```
 
+### Launch the Docs Agent web app as a widget
+
+The command below launches the Docs Agent web app to use
+a widget-friendly template:
+
+```sh
+agent chatbot --app_mode widget
+```
+
+### Launch the Docs Agent web app with a log view enabled
+
+The command below launches the Docs Agent web app while enabling
+a log view page (which is accessible at `<APP_URL>/logs`):
+
+```sh
+agent chatbot --enable_show_logs
+```
+
 ## Docs Agent benchmark test
 
 ### Run the Docs Agent benchmark test
@@ -106,6 +142,22 @@ You may also specify multiple products, for example:
 agent tellme which modules are available? --product=Flutter --product=Angular --product=Android
 ```
 
+### Ask for advice
+
+The command below reads a request and a filename from the arguments,
+asks the Gemini model, and prints its response:
+
+```sh
+agent helpme <REQUEST> --file <PATH_TO_FILE>
+```
+
+Replace `REQUEST` with a prompt and `PATH_TO_FILE` with a file's
+absolure or relative path, for example:
+
+```sh
+agent helpme write comments for this C++ file? --file ../my-project/test.cc
+```
+
 ## Online corpus management
 
 ### List all existing online corpora