Merge remote-tracking branch 'origin/main' into log-analysis

Author: CTY-git

.github/workflows/test.yml

@@ -1,10 +1,6 @@
 name: Tests
 
 on:
-  pull_request:
-    types:
-      - ready_for_review
-      - review_requested
   push:
     branches-ignore:
       - main
@@ -15,15 +11,12 @@ on:
       - generateunittests-*
       - generatecodeusageexample-*
       - resolveissue-*
-
       - demo*
 
 # Credits to https://blog.maximeheckel.com/posts/building-perfect-github-action-frontend-teams/#you-are-terminated
 concurrency:
   # Here the group is defined by the head_ref of the PR
   group: ${{ github.head_ref || github.ref_name }}
-  # Here we specify that we'll cancel any "in progress" workflow of the same group. Thus if we push, ammend a commit and push
-  # again the previous workflow will be cancelled, thus saving us github action build minutes and avoid any conflicts
   cancel-in-progress: true
 
 jobs:
@@ -211,7 +204,7 @@ jobs:
         run: |
           source .venv/bin/activate
           patchwork PRReview --log debug \
-          --patched_api_key=${{ secrets.PATCHED_API_KEY }} \
+          --anthropic_api_key=${{ secrets.ANTHROPIC_API_KEY }} \
           --github_api_key=${{ secrets.SCM_GITHUB_KEY }} \
           --pr_url=https://github.com/patched-codes/patchwork/pull/${{ steps.findPr.outputs.number }} \
           --disable_telemetry

patchwork/common/client/llm/openai_.py

@@ -14,6 +14,7 @@
 from typing_extensions import Dict, Iterable, List, Optional, Union
 
 from patchwork.common.client.llm.protocol import NOT_GIVEN, LlmClient, NotGiven
+from patchwork.logger import logger
 
 
 @functools.lru_cache
@@ -36,6 +37,7 @@ class OpenAiLlmClient(LlmClient):
         "o1-mini": 128_000,
         "gpt-4o-mini": 128_000,
         "gpt-4o": 128_000,
+        "o3-mini": 128_000,
     }
 
     def __init__(self, api_key: str, base_url=None, **kwargs):
@@ -87,7 +89,12 @@ def is_prompt_supported(
 
         model_limit = self.__get_model_limits(model)
         token_count = 0
-        encoding = tiktoken.encoding_for_model(model)
+        encoding = None
+        try:
+            encoding = tiktoken.encoding_for_model(model)
+        except Exception as e:
+            logger.error(f"Error getting encoding for model {model}: {e}, using gpt-4o as fallback")
+            encoding = tiktoken.encoding_for_model("gpt-4o")
         for message in messages:
             message_token_count = len(encoding.encode(message.get("content")))
             token_count = token_count + message_token_count

patchwork/patchflows/PRReview/PRReview.py

@@ -1,124 +1,144 @@
-import json
 from pathlib import Path
 
 import yaml
 
 from patchwork.common.utils.progress_bar import PatchflowProgressBar
 from patchwork.common.utils.step_typing import validate_steps_with_inputs
 from patchwork.step import Step
-from patchwork.steps import (
-    LLM,
-    CallLLM,
-    CreatePRComment,
-    ExtractModelResponse,
-    PreparePR,
-    PreparePrompt,
-    ReadPRDiffs,
-)
+from patchwork.steps import CreatePRComment, ReadPRDiffs, SimplifiedLLMOnce
 
 _DEFAULT_PROMPT_JSON = Path(__file__).parent / "pr_review_prompt.json"
 _DEFAULT_INPUT_FILE = Path(__file__).parent / "defaults.yml"
 
 
-_NONE = "none"
-_SHORT = "short"
-_LONG = "long"
-_SUMMARY_LEVEL = {
-    _NONE: 0,
-    _SHORT: 1,
-    _LONG: 2,
-}
-
-
 class PRReview(Step):
     def __init__(self, inputs: dict):
         PatchflowProgressBar(self).register_steps(
-            CallLLM,
-            CreatePRComment,
-            ExtractModelResponse,
-            PreparePR,
-            PreparePrompt,
             ReadPRDiffs,
+            SimplifiedLLMOnce,
+            CreatePRComment,
         )
         final_inputs = yaml.safe_load(_DEFAULT_INPUT_FILE.read_text())
         final_inputs.update(inputs)
 
-        if "prompt_template_file" not in final_inputs.keys():
-            final_inputs["prompt_template_file"] = _DEFAULT_PROMPT_JSON
-
-        diff_summary = final_inputs.get("diff_summary", _LONG)
-        if diff_summary.lower() not in _SUMMARY_LEVEL.keys():
-            raise ValueError(f"Invalid diff_summary, accepted diff_summary values: {_SUMMARY_LEVEL.keys()}")
-        self.verbosity = _SUMMARY_LEVEL[diff_summary.lower()]
-
-        self.is_suggestion_required = bool(final_inputs.get("diff_suggestion"))
-
         validate_steps_with_inputs(
             set(final_inputs.keys()).union(
                 {
-                    "prompt_id",
-                    "prompt_values",
-                    "modified_code_files",
+                    "user_prompt",
+                    "prompt_value",
+                    "json_schema",
                     "pr_comment",
                 }
             ),
             ReadPRDiffs,
-            LLM,
-            PreparePR,
+            SimplifiedLLMOnce,
             CreatePRComment,
         )
 
         self.inputs = final_inputs
 
     def run(self) -> dict:
-        if self.verbosity == _SUMMARY_LEVEL[_NONE]:
-            return dict()
-
-        outputs = ReadPRDiffs(self.inputs).run()
-        self.inputs["prompt_values"] = outputs["diffs"]
-
-        outputs = LLM(
-            dict(
-                prompt_id="diffreview-suggestion" if self.is_suggestion_required else "diffreview",
-                model_response_format=dict(type="json_object"),
-                **self.inputs,
-            )
-        ).run()
-        self.inputs.update(outputs)
-
-        summaries = []
-        for raw_response, prompt_values in zip(self.inputs["openai_responses"], self.inputs["prompt_values"]):
-            response = json.loads(raw_response)
-            summary = {}
-            if "path" in prompt_values.keys():
-                summary["path"] = prompt_values["path"]
-            if "review" in response.keys():
-                summary["commit_message"] = response["review"]
-            if "suggestion" in response.keys():
-                summary["patch_message"] = response["suggestion"]
-            summaries.append(summary)
-
-        header = ""
-        if self.verbosity > _SUMMARY_LEVEL[_SHORT]:
-            filtered_summaries = [
-                str(summary["commit_message"]) for summary in summaries if summary.get("commit_message")
-            ]
-            self.inputs["prompt_id"] = "diffreview_summary"
-            self.inputs["prompt_values"] = [{"diffreviews": "\n".join(filtered_summaries)}]
-
-            outputs = PreparePrompt(self.inputs).run()
-            self.inputs.update(outputs)
-            outputs = CallLLM(self.inputs).run()
-            self.inputs.update(outputs)
-            header = self.inputs["openai_responses"][0]
-
-        self.inputs["pr_header"] = header
-        self.inputs["modified_code_files"] = summaries
-        outputs = PreparePR(self.inputs).run()
-        self.inputs.update(outputs)
-
-        self.inputs["pr_comment"] = self.inputs["pr_body"]
-        outputs = CreatePRComment(self.inputs).run()
-        self.inputs.update(outputs)
-
-        return self.inputs
+        pr_diffs_outputs = ReadPRDiffs(self.inputs).run()
+
+        reviews = []
+        for diffs in iter(pr_diffs_outputs["diffs"]):
+            llm1_outputs = SimplifiedLLMOnce(
+                dict(
+                    prompt_value=diffs,
+                    user_prompt="""\
+Analyze the following code diff against the provided rules:
+
+<CODE_DIFF>
+{{diff}}
+</CODE_DIFF>
+
+<RULES>
+- Do not ignore potential bugs in the code.
+- Do not overlook possible security vulnerabilities introduced by code modifications.
+- Do not deviate from the original coding standards established in the pull request.
+</RULES>
+
+For each rule, determine if there\'s a violation. Use the following chain of thought process:
+
+1. Understand the rule
+2. Examine the diff line by line
+3. Identify any potential violations
+4. Determine the specific line numbers of violations
+5. Summarize your findings
+
+Rule 1:
+1. Rule understanding: [Briefly explain the rule]
+2. Diff examination: [Describe how you\'re examining the diff]
+3. Potential violations: [List any potential violations you\'ve identified]
+4. Line numbers: [If violations exist, list the specific line numbers]
+5. Summary: [Summarize your findings for this rule]
+
+Rule 2:
+[Repeat the above structure for each rule]
+
+Now, carefully review your reasoning in the section above. Ensure that your conclusions are consistent with the analysis you\'ve done for each rule.
+
+Your review should have the following markdown format:
+
+<REVIEW_FORMAT>
+## File Changed: `{{path}}`
+
+Details: [If rule violation include brief prescriptive explanation]
+
+Affected Code Snippet: 
+[Original code enclosed in a code block from the file that is affected by this violation. If no violation, write "N/A"]
+
+Start Line: [Starting Line number of the affected code. If no violation, write "N/A"]
+
+End Line: [Ending Line number of the affected code. If no violation, write "N/A"]
+
+-------------
+
+Details: [If rule violation include brief prescriptive explanation]
+
+Affected Code Snippet: 
+[Original code enclosed in a code block from the file that is affected by this violation. If no violation, write "N/A"]
+
+Start Line: [Starting Line number of the affected code. If no violation, write "N/A"]
+
+End Line: [Ending Line number of the affected code. If no violation, write "N/A"]
+
+-------------
+
+... (continue for all rules)
+</REVIEW_FORMAT>
+
+Ensure that you include all rules in your response, even if there\'s no violation. The output should directly reflect the reasoning in your thinking section.
+""",
+                    json_schema={"review": "The markdown text of the reviews"},
+                    **self.inputs,
+                )
+            ).run()
+
+            llm2_outputs = SimplifiedLLMOnce(
+                dict(
+                    prompt_value=llm1_outputs,
+                    user_prompt="""\
+You are a software manager compiling code reviews from all teams. You are given a list of code reviews. You have to remove code reviews that is either not actionable or useful. Do not change the accepted reviews, return the original review for the response. Do not remove the path from the review.
+
+<code_reviews>
+{{review}}
+<code_reviews>
+
+You should return an empty response if there are no code reviews that is actionable or useful.
+""",
+                    json_schema={"review": "The reviews curated"},
+                    **self.inputs,
+                )
+            ).run()
+
+            review = llm2_outputs.get("review")
+            if review is not None and len(str(review)) > 0:
+                reviews.append(review)
+
+        if len(reviews) > 0:
+            reviews_str = "\n".join(reviews)
+        else:
+            reviews_str = "No issues found."
+
+        return CreatePRComment(dict(pr_comment=reviews_str, **self.inputs)).run()

patchwork/patchflows/PRReview/defaults.yml

@@ -1,7 +1,4 @@
 # PRReview Inputs
-diff_summary: long
-diff_suggestion: false
-
 
 # ReadPRDiffs Inputs
 # github_api_key: required-for-github-scm
@@ -14,7 +11,7 @@ diff_suggestion: false
 # CallLLM Inputs
 # openai_api_key: required-for-chatgpt
 # google_api_key: required-for-gemini
-# model: gpt-4o
+ model: claude-3-5-sonnet-latest
 # client_base_url: https://api.openai.com/v1
 # Example HF model
 # client_base_url: https://api-inference.huggingface.co/models/codellama/CodeLlama-70b-Instruct-hf/v1