incrementally summarize (#211)

Author: harjotgill

README.md

@@ -1,10 +1,10 @@
-# OpenAI ChatGPT based PR reviewer and summarizer
+# OpenAI ChatGPT-based PR reviewer and summarizer
 
 ![AI](./docs/images/ai.png)
 
 ## Overview
 
-This [OpenAI ChatGPT](https://platform.openai.com/docs/guides/chat) based GitHub
+This [OpenAI ChatGPT-based](https://platform.openai.com/docs/guides/chat) GitHub
 Action provides a summary, release notes and review of pull requests. The
 prompts have been tuned for a concise response. To prevent excessive
 notifications, this action can be configured to skip adding review comments when
@@ -22,6 +22,8 @@ NOTES:
   has a
   [more conservative data usage policy](https://openai.com/policies/api-data-usage-policies)
   compared to their ChatGPT offering.
+- This action is not affiliated with OpenAI.
+- This action is going through rapid iteration, so expect breaking changes.
 
 ## Usage
 
@@ -178,7 +180,7 @@ $ npm run build && npm run package
 
 ## FAQs
 
-### Review pull request from forks
+### Review pull requests from forks
 
 GitHub Actions limits the access of secrets from forked repositories. To enable
 this feature, you need to use the `pull_request_target` event instead of

action.yml

@@ -9,14 +9,11 @@ inputs:
     required: false
     description: 'Enable debug mode'
     default: 'false'
-  max_files_to_summarize:
+  max_files:
     required: false
     description:
-      'Max files to summarize. Less than or equal to 0 means no limit.'
-    default: '40'
-  max_files_to_review:
-    required: false
-    description: 'Max files to review. Less than or equal to 0 means no limit.'
+      'Max files to summarize and review. Less than or equal to 0 means no
+      limit.'
     default: '0'
   review_comment_lgtm:
     required: false
@@ -109,7 +106,7 @@ inputs:
       disregarding minor issues.
   summarize_file_diff:
     required: false
-    description: 'The prompt for file'
+    description: 'The prompt for summarizing a file'
     default: |
       GitHub pull request title: 
       `$title` 
@@ -124,7 +121,7 @@ inputs:
       $file_content
       ```
 
-      Diff:
+      Diff for `$filename`:
       ```diff
       $file_diff
       ```
@@ -134,17 +131,17 @@ inputs:
     required: false
     description: 'The prompt for final summarization response'
     default: |
-      Here is the summary of changes you have generated for each file:
+      Here is the summary of changes you have generated for files:
       ```
-      $summary
+      $raw_summary
       ```
 
       Provide your final response in the `markdown` format with 
       the following content:
         - High-level summary (comment on the overall change instead of 
           specific files within 80 words)
         - Table of files and their summaries. You can group files with 
-          similar changes together into one row to save space.
+          similar changes together into a single row to save space.
 
       Avoid additional commentary as this summary will be added as a 
       comment on the GitHub pull request.

dist/index.js

@@ -24,6 +24,11 @@ export const DESCRIPTION_TAG =
 export const DESCRIPTION_TAG_END =
   '<!-- end of auto-generated comment: release notes by openai -->'
 
+export const RAW_SUMMARY_TAG =
+  '<!-- This is an auto-generated comment: raw summary by openai -->'
+export const RAW_SUMMARY_TAG_END =
+  '<!-- end of auto-generated comment: raw summary by openai -->'
+
 export class Commenter {
   /**
    * @param mode Can be "create", "replace". Default is "replace".
@@ -61,32 +66,52 @@ ${tag}`
     }
   }
 
-  get_description(description: string) {
-    // remove our summary from description by looking for description_tag and description_tag_end
-    const start = description.indexOf(DESCRIPTION_TAG)
-    const end = description.indexOf(DESCRIPTION_TAG_END)
+  getContentWithinTags(content: string, startTag: string, endTag: string) {
+    const start = content.indexOf(startTag)
+    const end = content.indexOf(endTag)
     if (start >= 0 && end >= 0) {
-      return (
-        description.slice(0, start) +
-        description.slice(end + DESCRIPTION_TAG_END.length)
-      )
+      return content.slice(start + startTag.length, end)
     }
-    return description
+    return ''
   }
 
-  get_release_notes(description: string) {
-    // get our summary from description by looking for description_tag and description_tag_end
-    // and remove any content within that which is in markdown quote (>)
-    const start = description.indexOf(DESCRIPTION_TAG)
-    const end = description.indexOf(DESCRIPTION_TAG_END)
+  removeContentWithinTags(content: string, startTag: string, endTag: string) {
+    const start = content.indexOf(startTag)
+    const end = content.indexOf(endTag)
     if (start >= 0 && end >= 0) {
-      const release_notes = description.slice(
-        start + DESCRIPTION_TAG.length,
-        end
-      )
-      return release_notes.replace(/(^|\n)> .*/g, '')
+      return content.slice(0, start) + content.slice(end + endTag.length)
     }
-    return ''
+    return content
+  }
+
+  get_raw_summary(summary: string) {
+    const content = this.getContentWithinTags(
+      summary,
+      RAW_SUMMARY_TAG,
+      RAW_SUMMARY_TAG_END
+    )
+    // remove the first and last line
+    const lines = content.split('\n')
+    lines.shift()
+    lines.pop()
+    return lines.join('\n')
+  }
+
+  get_description(description: string) {
+    return this.removeContentWithinTags(
+      description,
+      DESCRIPTION_TAG,
+      DESCRIPTION_TAG_END
+    )
+  }
+
+  get_release_notes(description: string) {
+    const release_notes = this.getContentWithinTags(
+      description,
+      DESCRIPTION_TAG,
+      DESCRIPTION_TAG_END
+    )
+    return release_notes.replace(/(^|\n)> .*/g, '')
   }
 
   async update_description(pull_number: number, message: string) {

src/commenter.ts

@@ -8,8 +8,7 @@ async function run(): Promise<void> {
   const options: Options = new Options(
     core.getBooleanInput('debug'),
     core.getBooleanInput('summary_only'),
-    core.getInput('max_files_to_summarize'),
-    core.getInput('max_files_to_review'),
+    core.getInput('max_files'),
     core.getBooleanInput('review_comment_lgtm'),
     core.getMultilineInput('path_filters'),
     core.getInput('system_message'),

src/main.ts

@@ -47,7 +47,7 @@ export class Inputs {
   system_message: string
   title: string
   description: string
-  summary: string
+  raw_summary: string
   release_notes: string
   filename: string
   file_content: string
@@ -74,7 +74,7 @@ export class Inputs {
     this.system_message = system_message
     this.title = title
     this.description = description
-    this.summary = summary
+    this.raw_summary = summary
     this.release_notes = release_notes
     this.filename = filename
     this.file_content = file_content
@@ -90,7 +90,7 @@ export class Inputs {
       this.system_message,
       this.title,
       this.description,
-      this.summary,
+      this.raw_summary,
       this.release_notes,
       this.filename,
       this.file_content,
@@ -115,8 +115,8 @@ export class Inputs {
     if (this.description) {
       content = content.replace('$description', this.description)
     }
-    if (this.summary) {
-      content = content.replace('$summary', this.summary)
+    if (this.raw_summary) {
+      content = content.replace('$raw_summary', this.raw_summary)
     }
     if (this.release_notes) {
       content = content.replace('$release_notes', this.release_notes)
@@ -186,8 +186,7 @@ export class OpenAIOptions {
 export class Options {
   debug: boolean
   summary_only: boolean
-  max_files_to_summarize: number
-  max_files_to_review: number
+  max_files: number
   review_comment_lgtm: boolean
   path_filters: PathFilter
   system_message: string
@@ -203,8 +202,7 @@ export class Options {
   constructor(
     debug: boolean,
     summary_only: boolean,
-    max_files_to_summarize = '40',
-    max_files_to_review = '0',
+    max_files = '0',
     review_comment_lgtm = false,
     path_filters: string[] | null = null,
     system_message = '',
@@ -217,8 +215,7 @@ export class Options {
   ) {
     this.debug = debug
     this.summary_only = summary_only
-    this.max_files_to_summarize = parseInt(max_files_to_summarize)
-    this.max_files_to_review = parseInt(max_files_to_review)
+    this.max_files = parseInt(max_files)
     this.review_comment_lgtm = review_comment_lgtm
     this.path_filters = new PathFilter(path_filters)
     this.system_message = system_message
@@ -236,8 +233,7 @@ export class Options {
   print(): void {
     core.info(`debug: ${this.debug}`)
     core.info(`summary_only: ${this.summary_only}`)
-    core.info(`max_files_to_summarize: ${this.max_files_to_summarize}`)
-    core.info(`max_files_to_review: ${this.max_files_to_review}`)
+    core.info(`max_files: ${this.max_files}`)
     core.info(`review_comment_lgtm: ${this.review_comment_lgtm}`)
     core.info(`path_filters: ${this.path_filters}`)
     core.info(`system_message: ${this.system_message}`)

src/options.ts

@@ -5,7 +5,6 @@ import {
   Commenter,
   COMMENT_REPLY_TAG,
   COMMENT_TAG,
-  EXTRA_CONTENT_TAG,
   SUMMARIZE_TAG
 } from './commenter.js'
 import {octokit} from './octokit.js'
@@ -147,16 +146,6 @@ export const handleReviewComment = async (
         }
       }
 
-      // get summary of the PR
-      const summary = await commenter.find_comment_with_tag(
-        SUMMARIZE_TAG,
-        pull_number
-      )
-      if (summary) {
-        // remove all content below EXTRA_CONTENT_TAG
-        inputs.summary = summary.body.split(EXTRA_CONTENT_TAG)[0]
-      }
-
       // get tokens so far
       let tokens = tokenizer.get_token_count(prompts.render_comment(inputs))
 
@@ -168,7 +157,6 @@ export const handleReviewComment = async (
         )
         return
       }
-
       // pack file content and diff into the inputs if they are not too long
       if (file_content.length > 0) {
         // count occurrences of $file_content in prompt
@@ -199,6 +187,24 @@ export const handleReviewComment = async (
         }
       }
 
+      // get summary of the PR
+      const summary = await commenter.find_comment_with_tag(
+        SUMMARIZE_TAG,
+        pull_number
+      )
+      if (summary) {
+        // pack summary into the inputs if it is not too long
+        const raw_summary = commenter.get_raw_summary(summary.body)
+        const summary_tokens = tokenizer.get_token_count(raw_summary)
+        if (
+          tokens + summary_tokens <=
+          options.heavy_token_limits.request_tokens
+        ) {
+          tokens += summary_tokens
+          inputs.raw_summary = raw_summary
+        }
+      }
+
       const [reply] = await heavyBot.chat(prompts.render_comment(inputs), {})
 
       await commenter.review_comment_reply(pull_number, topLevelComment, reply)