saga4/fix duplication suggestion issue

code_to_optimize/few_formatting_errors.py

@@ -0,0 +1,47 @@
+import os
+
+class UnformattedExampleClass(object):
+    def __init__(
+        self,
+        name,
+        age=    None,
+        email=  None,
+        phone=None,
+        address=None,
+        city=None,
+        state=None,
+        zip_code=None,
+    ):
+        self.name = name
+        self.age = age
+        self.email = email
+        self.phone = phone
+        self.   address = address
+        self.city = city
+        self.state = state
+        self.zip_code = zip_code
+        self.data = {"name": name, "age": age, "email": email}
+
+    def get_info(self):
+        return f"Name: {self.name}, Age: {self.age}"
+
+    def update_data(self, **kwargs):
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        self.data.update(kwargs)
+
+
+def process_data(
+    data_list, filter_func=None, transform_func=None, sort_key=None, reverse=False
+):
+    if not data_list:
+        return []
+    if filter_func:
+        data_list = [   item for item in data_list if filter_func(item)]
+    if transform_func:
+        data_list = [transform_func(item) for item in data_list]
+    if sort_key:
+        data_list = sorted(data_list, key=sort_key, reverse=reverse)
+    return data_list
+

code_to_optimize/many_formatting_errors.py

@@ -0,0 +1,147 @@
+import os,sys,json,datetime,math,random;import requests;from collections import defaultdict,OrderedDict
+from typing import List,Dict,Optional,Union,Tuple,Any;import numpy as np;import pandas as pd
+
+# This is a poorly formatted Python file with many style violations
+
+class   UnformattedExampleClass( object ):
+    def __init__(self,name,age=None,email=None,phone=None,address=None,city=None,state=None,zip_code=None):
+        self.name=name;self.age=age;self.email=email;self.phone=phone
+        self.address=address;self.city=city;self.state=state;self.zip_code=zip_code
+        self.data={"name":name,"age":age,"email":email}
+
+    def   get_info(self   ):
+        return f"Name: {self.name}, Age: {self.age}"
+
+    def update_data(self,**kwargs):
+        for key,value in kwargs.items():
+            if hasattr(self,key):setattr(self,key,value)
+        self.data.update(kwargs)
+
+def process_data(data_list,filter_func=None,transform_func=None,sort_key=None,reverse=False):
+    if not data_list:return[]
+    if filter_func:data_list=[item for item in data_list if filter_func(item)]
+    if transform_func:data_list=[transform_func(item)for item in data_list]
+    if sort_key:data_list=sorted(data_list,key=sort_key,reverse=reverse)
+    return data_list
+
+def calculate_statistics(numbers):
+    if not numbers:return None
+    mean=sum(numbers)/len(numbers);         median=sorted(numbers)[len(numbers)//2]
+    variance=sum((x-mean)**2 for x in numbers)/len(numbers);std_dev=math.sqrt(variance)
+    return      {"mean":mean,"median":median,"variance":variance,"std_dev":std_dev,"min":min(numbers),"max":max(numbers)}
+
+def complex_nested_function(x,y,z):
+    def inner_function_1(a,b):
+        def deeply_nested(c,d):
+            return c*d+a*b
+        return deeply_nested(a+1,b-1)+deeply_nested(a-1,b+1)
+    def     inner_function_2    (a,b,c):
+        result=[]
+        for i in range(a):
+            for j in     range(b):
+                for k in range(c):
+                    if i*j*k>0:result.append(i*j*k)
+                    elif i+j+k==0:result.append(-1)
+                    else    :result.append(0)
+        return result
+    return inner_function_1(x,y)+sum(inner_function_2(x,y,z))
+
+# Long lines and poor dictionary formatting
+user_data={"users":[{"id":1,"name":"John Doe","email":"[email protected]","preferences":{"theme":"dark","notifications":True,"language":"en"},"metadata":{"created_at":"2023-01-01","last_login":"2024-01-01","login_count":150}},{"id":2,"name":"Jane Smith","email":"[email protected]","preferences":{"theme":"light","notifications":False,"language":"es"},"metadata":{"created_at":"2023-02-15","last_login":"2024-01-15","login_count":89}}]}
+
+# Poor list formatting and string concatenation
+long_list_of_items=['item_1','item_2','item_3','item_4','item_5','item_6','item_7','item_8','item_9','item_10','item_11','item_12','item_13','item_14','item_15','item_16','item_17','item_18','item_19','item_20']
+
+def generate_report(data,include_stats=True,include_charts=False,format_type='json',output_file=None):
+    if not data:raise ValueError("Data cannot be empty")
+    report={'timestamp':datetime.datetime.now().isoformat(),'data_count':len(data),'summary':{}}
+
+    # Bad formatting in loops and conditionals
+    for i,item in enumerate(data):
+        if isinstance(item,dict):
+            for key,value in item.items():
+                if key not in report['summary']:report['summary'][key]=[]
+                report['summary'][key].append(value)
+        elif isinstance(item,(int,float)):
+            if 'numbers' not in report['summary']:report['summary']['numbers']=[]
+            report['summary']['numbers'].append(item)
+        else:
+            if 'other' not in report['summary']:report['summary']['other']=[]
+            report['summary']['other'].append(str(item))
+
+    if include_stats and 'numbers' in report['summary']:
+        numbers=report['summary']['numbers']
+        report['statistics']=calculate_statistics(numbers)
+
+    # Long conditional chain with poor formatting
+    if format_type=='json':result=json.dumps(report,indent=None,separators=(',',':'))
+    elif format_type=='pretty_json':result=json.dumps(report,indent=2)
+    elif format_type=='string':result=str(report)
+    else:result=report
+
+    if output_file:
+        with open(output_file,'w')as f:f.write(result if isinstance(result,str)else json.dumps(result))
+
+    return result
+
+class   DataProcessor  (  UnformattedExampleClass  )  :
+    def __init__(self,data_source,config=None,debug=False):
+        super().__init__("DataProcessor")
+        self.data_source=data_source;self.config=config or{};self.debug=debug
+        self.processed_data=[];self.errors=[];self.warnings=[]
+
+    def   load_data  (  self  )  :
+        try:
+            if isinstance(self.data_source,str):
+                if self.data_source.endswith('.json'):
+                    with open(self.data_source,'r')as f:data=json.load(f)
+                elif self.data_source.endswith('.csv'):data=pd.read_csv(self.data_source).to_dict('records')
+                else:raise ValueError(f"Unsupported file type: {self.data_source}")
+            elif isinstance(self.data_source,list):data=self.data_source
+            else:data=[self.data_source]
+            return data
+        except Exception as e:
+            self.errors.append(str(e));return[]
+
+    def validate_data(self,data):
+        valid_items=[];invalid_items=[]
+        for item in data:
+            if isinstance(item,dict)and'id'in item and'name'in item:valid_items.append(item)
+            else:invalid_items.append(item)
+        if invalid_items:self.warnings.append(f"Found {len(invalid_items)} invalid items")
+        return valid_items
+
+    def process(self):
+        data=self.load_data()
+        if not data:return{"success":False,"error":"No data loaded"}
+
+        validated_data=self.validate_data(data)
+        processed_result=process_data(validated_data,
+                                    filter_func=lambda x:x.get('active',True),
+                                    transform_func=lambda x:{**x,'processed_at':datetime.datetime.now().isoformat()},
+                                    sort_key=lambda x:x.get('name',''))
+
+        self.processed_data=processed_result
+        return{"success":True,"count":len(processed_result),"data":processed_result}
+if __name__=="__main__":
+    sample_data=[{"id":1,"name":"Alice","active":True},{"id":2,"name":"Bob","active":False},{"id":3,"name":"Charlie","active":True}]
+
+    processor=DataProcessor(sample_data,config={"debug":True})
+    result=processor.process()
+
+    if result["success"]:
+        print(f"Successfully processed {result['count']} items")
+        for item in result["data"][:3]:print(f"- {item['name']} (ID: {item['id']})")
+    else:print(f"Processing failed: {result.get('error','Unknown error')}")
+
+    # Generate report with poor formatting
+    report=generate_report(sample_data,include_stats=True,format_type='pretty_json')
+    print("Generated report:",report[:100]+"..."if len(report)>100 else report)
+
+    # Complex calculation with poor spacing
+    numbers=[random.randint(1,100)for _ in range(50)]
+    stats=calculate_statistics(numbers)
+    complex_result=complex_nested_function(5,3,2)
+
+    print(f"Statistics: mean={stats['mean']:.2f}, std_dev={stats['std_dev']:.2f}")
+    print(f"Complex calculation result: {complex_result}")

code_to_optimize/no_formatting_errors.py

@@ -0,0 +1,71 @@
+import os, sys, json, datetime, math, random
+import requests
+from collections import defaultdict, OrderedDict
+from typing import List, Dict, Optional, Union, Tuple, Any
+import numpy as np
+import pandas as pd
+
+# This is a poorly formatted Python file with many style violations
+
+
+class UnformattedExampleClass(object):
+    def __init__(
+        self,
+        name,
+        age=None,
+        email=None,
+        phone=None,
+        address=None,
+        city=None,
+        state=None,
+        zip_code=None,
+    ):
+        self.name = name
+        self.age = age
+        self.email = email
+        self.phone = phone
+        self.address = address
+        self.city = city
+        self.state = state
+        self.zip_code = zip_code
+        self.data = {"name": name, "age": age, "email": email}
+
+    def get_info(self):
+        return f"Name: {self.name}, Age: {self.age}"
+
+    def update_data(self, **kwargs):
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+        self.data.update(kwargs)
+
+
+def process_data(
+    data_list, filter_func=None, transform_func=None, sort_key=None, reverse=False
+):
+    if not data_list:
+        return []
+    if filter_func:
+        data_list = [item for item in data_list if filter_func(item)]
+    if transform_func:
+        data_list = [transform_func(item) for item in data_list]
+    if sort_key:
+        data_list = sorted(data_list, key=sort_key, reverse=reverse)
+    return data_list
+
+
+def calculate_statistics(numbers):
+    if not numbers:
+        return None
+    mean = sum(numbers) / len(numbers)
+    median = sorted(numbers)[len(numbers) // 2]
+    variance = sum((x - mean) ** 2 for x in numbers) / len(numbers)
+    std_dev = math.sqrt(variance)
+    return {
+        "mean": mean,
+        "median": median,
+        "variance": variance,
+        "std_dev": std_dev,
+        "min": min(numbers),
+        "max": max(numbers),
+    }

codeflash/code_utils/formatter.py

@@ -3,7 +3,7 @@
 import os
 import shlex
 import subprocess
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 import isort
 
@@ -13,14 +13,53 @@
     from pathlib import Path
 
 
+def get_diff_output_by_black(filepath: str, unformatted_content: str) -> Optional[str]:
+    try:
+        from black import Mode, format_file_contents, output, report
+
+        formatted_content = format_file_contents(src_contents=unformatted_content, fast=True, mode=Mode())
+        return output.diff(unformatted_content, formatted_content, a_name=filepath, b_name=filepath)
+    except (ImportError, report.NothingChanged):
+        return None
+
+
+def get_diff_lines_count(diff_output: str) -> int:
+    lines = diff_output.split("\n")
+
+    def is_diff_line(line: str) -> bool:
+        return line.startswith(("+", "-")) and not line.startswith(("+++", "---"))
+
+    diff_lines = [line for line in lines if is_diff_line(line)]
+    return len(diff_lines)
+
+
+def is_safe_to_format(filepath: str, content: str, max_diff_lines: int = 100) -> bool:
+    diff_changes_str = None
+
+    diff_changes_str = get_diff_output_by_black(filepath, unformatted_content=content)
+
+    if diff_changes_str is None:
+        logger.warning("Looks like black formatter not found, make sure it is installed.")
+        return False
+
+    diff_lines_count = get_diff_lines_count(diff_changes_str)
+    if diff_lines_count > max_diff_lines:
+        logger.debug(f"Skipping formatting {filepath}: {diff_lines_count} lines would change (max: {max_diff_lines})")
+        return False
+
+    return True
+
+
 def format_code(formatter_cmds: list[str], path: Path, print_status: bool = True) -> str:  # noqa
     # TODO: Only allow a particular whitelist of formatters here to prevent arbitrary code execution
     formatter_name = formatter_cmds[0].lower()
     if not path.exists():
         msg = f"File {path} does not exist. Cannot format the file."
         raise FileNotFoundError(msg)
-    if formatter_name == "disabled":
-        return path.read_text(encoding="utf8")
+    file_content = path.read_text(encoding="utf8")
+    if formatter_name == "disabled" or not is_safe_to_format(filepath=str(path), content=file_content):
+        return file_content
+
     file_token = "$file"  # noqa: S105
     for command in formatter_cmds:
         formatter_cmd_list = shlex.split(command, posix=os.name != "nt")
@@ -29,7 +68,7 @@ def format_code(formatter_cmds: list[str], path: Path, print_status: bool = True
             result = subprocess.run(formatter_cmd_list, capture_output=True, check=False)
             if result.returncode == 0:
                 if print_status:
-                    console.rule(f"Formatted Successfully with: {formatter_name.replace('$file', path.name)}")
+                    console.rule(f"Formatted Successfully with: {command.replace('$file', path.name)}")
             else:
                 logger.error(f"Failed to format code with {' '.join(formatter_cmd_list)}")
         except FileNotFoundError as e:

codeflash/discovery/functions_to_optimize.py

@@ -476,12 +476,14 @@ def filter_functions(
         if blocklist_funcs:
             functions_tmp = []
             for function in _functions:
-                if not (
+                if (
                     function.file_path.name in blocklist_funcs
                     and function.qualified_name in blocklist_funcs[function.file_path.name]
                 ):
+                    # This function is in blocklist, we can skip it
                     blocklist_funcs_removed_count += 1
                     continue
+                # This function is NOT in blocklist. we can keep it
                 functions_tmp.append(function)
             _functions = functions_tmp