Transformers should read and write bytes (#765)

* fix regex transformer to handle windows style endings

* change test utils

* change read/write to read_bytes

Author: clavedeluna

src/codemodder/codemodder.py

@@ -23,14 +23,6 @@
 from codemodder.semgrep import run as run_semgrep
 
 
-def update_code(file_path, new_code):
-    """
-    Write the `new_code` to the `file_path`
-    """
-    with open(file_path, "w", encoding="utf-8") as f:
-        f.write(new_code)
-
-
 def find_semgrep_results(
     context: CodemodExecutionContext,
     codemods: Sequence[BaseCodemod],

src/codemodder/codemods/libcst_transformer.py

@@ -24,8 +24,7 @@ def update_code(file_path, new_code):
     """
     Write the `new_code` to the `file_path`
     """
-    with open(file_path, "w", encoding="utf-8") as f:
-        f.write(new_code)
+    file_path.write_bytes(new_code.encode("utf-8"))
 
 
 class LibcstResultTransformer(BaseTransformer):
@@ -261,8 +260,7 @@ def apply(
 
         try:
             with file_context.timer.measure("parse"):
-                with open(file_path, "r", encoding="utf-8") as f:
-                    source_tree = cst.parse_module(f.read())
+                source_tree = cst.parse_module(file_path.read_bytes().decode("utf-8"))
         except Exception:
             file_context.add_failure(file_path, reason := "Failed to parse file")
             logger.exception("%s %s", reason, file_path)

src/codemodder/codemods/regex_transformer.py

@@ -34,8 +34,11 @@ def apply(
         changes = []
         updated_lines = []
 
-        with open(file_context.file_path, "r", encoding="utf-8") as f:
-            original_lines = f.readlines()
+        original_lines = (
+            file_context.file_path.read_bytes()
+            .decode("utf-8")
+            .splitlines(keepends=True)
+        )
 
         for lineno, line in enumerate(original_lines):
             # TODO: use results to filter out which lines to change
@@ -57,8 +60,7 @@ def apply(
         diff = create_diff(original_lines, updated_lines)
 
         if not context.dry_run:
-            with open(file_context.file_path, "w+", encoding="utf-8") as original:
-                original.writelines(updated_lines)
+            file_context.file_path.write_bytes("".join(updated_lines).encode("utf-8"))
 
         return ChangeSet(
             path=str(file_context.file_path.relative_to(context.directory)),

src/codemodder/codemods/test/utils.py

@@ -129,8 +129,7 @@ def assert_changes(self, root, file_path, input_code, expected, changes):
         except AssertionError:
             raise DiffError(expected_diff, changes.diff)
 
-        with open(file_path, "r", encoding="utf-8") as tmp_file:
-            output_code = tmp_file.read()
+        output_code = file_path.read_bytes().decode("utf-8")
 
         try:
             assert output_code == (format_expected := dedent(expected))

src/codemodder/codemods/xml_transformer.py

@@ -218,16 +218,19 @@ def apply(
                 return None
 
             new_lines = output_file.readlines()
-            with open(file_path, "r") as original:
-                # TODO there's a failure potential here for very large files
-                diff = create_diff(
-                    original.readlines(),
-                    new_lines,
-                )
+            # TODO there's a failure potential here for very large files
+            original_lines = (
+                file_context.file_path.read_bytes()
+                .decode("utf-8")
+                .splitlines(keepends=True)
+            )
+            diff = create_diff(
+                original_lines,
+                new_lines,
+            )
 
             if not context.dry_run:
-                with open(file_path, "w+") as original:
-                    original.writelines(new_lines)
+                file_context.file_path.write_bytes("".join(new_lines).encode("utf-8"))
 
             return ChangeSet(
                 path=str(file_path.relative_to(context.directory)),

tests/codemods/test_xml_transformer.py

@@ -1,24 +1,27 @@
 from pathlib import Path
 
+import mock
+
 from codemodder.codemods.xml_transformer import (
     ElementAttributeXMLTransformer,
     XMLTransformerPipeline,
 )
 from codemodder.context import CodemodExecutionContext
 from codemodder.file_context import FileContext
 
+FILE_PATH = mock.MagicMock()
+FILE_PATH.read_bytes.return_value = b"""<?xml version="1.0" encoding="utf-8"?>"""
+
 
 def test_transformer_failure(mocker, caplog):
     mocker.patch(
         "defusedxml.expatreader.DefusedExpatParser.parse",
         side_effect=Exception,
     )
-    file_context = FileContext(
-        Path("home"),
-        Path("test.xml"),
-    )
+    file_context = FileContext(parent_dir := Path("home"), FILE_PATH)
+
     execution_context = CodemodExecutionContext(
-        directory=mocker.MagicMock(),
+        directory=parent_dir,
         dry_run=True,
         verbose=False,
         registry=mocker.MagicMock(),
@@ -45,10 +48,7 @@ def test_transformer(mocker):
     )
     mocker.patch("builtins.open")
     mocker.patch("codemodder.codemods.xml_transformer.create_diff", return_value="diff")
-    file_context = FileContext(
-        parent_dir := Path("home"),
-        parent_dir / Path("test.xml"),
-    )
+    file_context = FileContext(parent_dir := Path("home"), FILE_PATH)
     execution_context = CodemodExecutionContext(
         directory=parent_dir,
         dry_run=True,

tests/test_libcst_transformer.py

@@ -13,6 +13,7 @@
 from core_codemods.sonar.results import SonarResult
 
 FILE_PATH = mock.MagicMock()
+FILE_PATH.read_bytes.return_value = b"# some code to codemod"
 DEFECTDOJO_RESULTS = [
     DefectDojoResult.from_result(
         {

tests/test_regex_transformer.py

@@ -70,3 +70,39 @@ def test_transformer(mocker, tmp_path_factory):
     assert changeset is not None
     assert code.read_text() == text.replace("hello", "bye")
     assert changeset.changes[0].lineNumber == 1
+
+
+def test_transformer_windows_carriage(mocker, tmp_path_factory):
+    base_dir = tmp_path_factory.mktemp("foo")
+    code = base_dir / "code.py"
+    text = (
+        b"Hello, world!\r\nThis is a test string with Windows-style line endings.\r\n"
+    )
+    code.write_bytes(text)
+
+    file_context = FileContext(
+        base_dir,
+        code,
+    )
+    execution_context = CodemodExecutionContext(
+        directory=base_dir,
+        dry_run=False,
+        verbose=False,
+        registry=mocker.MagicMock(),
+        providers=None,
+        repo_manager=mocker.MagicMock(),
+        path_include=[],
+        path_exclude=[],
+    )
+    pipeline = RegexTransformerPipeline(
+        pattern=r"world", replacement="Earth", change_description="testing"
+    )
+
+    changeset = pipeline.apply(
+        context=execution_context,
+        file_context=file_context,
+        results=None,
+    )
+    assert changeset is not None
+    assert code.read_bytes() == text.replace(b"world", b"Earth")
+    assert changeset.changes[0].lineNumber == 1