fix: UnicodeEncodeError on Windows and add py314 (#126)

* chore: update test.yml as python 3.14 is out

* chore: Update test.yml to pin windows to windows-2022

* fix: Unicode decoding error and wc command

* fix: 'wc' is not recognized as an internal or external command

* docs: revert changes to readme

* ci: add more version test on windows and macos

Author: shenxianpeng

.github/workflows/test.yml

@@ -28,10 +28,10 @@ jobs:
         nox -s docs
 
   test-windows:
-    runs-on: windows-latest
+    runs-on: windows-2022
     strategy:
       matrix:
-        python-version: ["3.9"] # just test on one version to save time
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
     - name: Checkout Code
@@ -69,7 +69,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python-version: ["3.9"] # just test on one version to save time
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
     - name: Checkout Code
@@ -108,7 +108,7 @@ jobs:
       pull-requests: write
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14.0-beta.1"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
     - name: Checkout Code

.gitignore

@@ -6,6 +6,6 @@ gitstats.egg-info
 gitstory.egg-info
 build/
 gitstats-report
-test-report
-test-report.json
+test-report*
+test-report*.json
 docs/build

gitstats/main.py

@@ -723,7 +723,7 @@ def run(gitpath, outputpath, extra_fmt=None) -> int:
             import json
 
             print(f'Generating JSON file: "{output_file}"')
-            with open(output_file, "w") as file:
+            with open(output_file, "w", encoding="utf-8") as file:
                 json.dump(data.__dict__, file, default=str)
         else:
             print(f"Error: Unsupported format '{extra_fmt}'")

gitstats/report_creator.py

@@ -58,7 +58,7 @@ def create(self, data, path):
         self.create_graphs(path)
 
     def create_index_html(self, data, path):
-        f = open(path + "/index.html", "w")
+        f = open(path + "/index.html", "w", encoding="utf-8")
         format = "%Y-%m-%d %H:%M:%S"
         self.print_header(f)
 
@@ -124,7 +124,7 @@ def create_index_html(self, data, path):
     def create_activity_html(self, data, path):
         ###
         # Activity
-        f = open(path + "/activity.html", "w")
+        f = open(path + "/activity.html", "w", encoding="utf-8")
         self.print_header(f)
         f.write("<h1>Activity</h1>")
         self.print_nav(f)
@@ -179,7 +179,7 @@ def create_activity_html(self, data, path):
         for i in range(0, 24):
             f.write("<th>%d</th>" % i)
         f.write("</tr>\n<tr><th>Commits</th>")
-        fp = open(path + "/hour_of_day.dat", "w")
+        fp = open(path + "/hour_of_day.dat", "w", encoding="utf-8")
         for i in range(0, 24):
             if i in hour_of_day:
                 r = 127 + int(
@@ -209,7 +209,7 @@ def create_activity_html(self, data, path):
                 f.write("<td>0.00</td>")
         f.write("</tr></table>")
         f.write('<img src="hour_of_day.png" alt="Hour of Day">')
-        fg = open(path + "/hour_of_day.dat", "w")
+        fg = open(path + "/hour_of_day.dat", "w", encoding="utf-8")
         for i in range(0, 24):
             if i in hour_of_day:
                 fg.write("%d %d\n" % (i + 1, hour_of_day[i]))
@@ -222,7 +222,7 @@ def create_activity_html(self, data, path):
         day_of_week = data.get_activity_by_day_of_week()
         f.write('<div class="vtable"><table>')
         f.write("<tr><th>Day</th><th>Total (%)</th></tr>")
-        fp = open(path + "/day_of_week.dat", "w")
+        fp = open(path + "/day_of_week.dat", "w", encoding="utf-8")
         for d in range(0, 7):
             commits = 0
             if d in day_of_week:
@@ -275,7 +275,7 @@ def create_activity_html(self, data, path):
         f.write(html_header(2, "Month of Year"))
         f.write('<div class="vtable"><table>')
         f.write("<tr><th>Month</th><th>Commits (%)</th></tr>")
-        fp = open(path + "/month_of_year.dat", "w")
+        fp = open(path + "/month_of_year.dat", "w", encoding="utf-8")
         for mm in range(1, 13):
             commits = 0
             if mm in data.activity_by_month_of_year:
@@ -306,7 +306,7 @@ def create_activity_html(self, data, path):
             )
         f.write("</table></div>")
         f.write('<img src="commits_by_year_month.png" alt="Commits by year/month">')
-        fg = open(path + "/commits_by_year_month.dat", "w")
+        fg = open(path + "/commits_by_year_month.dat", "w", encoding="utf-8")
         for yymm in sorted(data.commits_by_month.keys()):
             fg.write("%s %s\n" % (yymm, data.commits_by_month[yymm]))
         fg.close()
@@ -330,7 +330,7 @@ def create_activity_html(self, data, path):
             )
         f.write("</table></div>")
         f.write('<img src="commits_by_year.png" alt="Commits by Year">')
-        fg = open(path + "/commits_by_year.dat", "w")
+        fg = open(path + "/commits_by_year.dat", "w", encoding="utf-8")
         for yy in sorted(data.commits_by_year.keys()):
             fg.write("%d %d\n" % (yy, data.commits_by_year[yy]))
         fg.close()
@@ -356,7 +356,7 @@ def create_activity_html(self, data, path):
     def create_authors_html(self, data, path):
         ###
         # Authors
-        f = open(path + "/authors.html", "w")
+        f = open(path + "/authors.html", "w", encoding="utf-8")
         self.print_header(f)
 
         f.write("<h1>Authors</h1>")
@@ -414,8 +414,8 @@ def create_authors_html(self, data, path):
                 % conf["max_authors"]
             )
 
-        fgl = open(path + "/lines_of_code_by_author.dat", "w")
-        fgc = open(path + "/commits_by_author.dat", "w")
+        fgl = open(path + "/lines_of_code_by_author.dat", "w", encoding="utf-8")
+        fgc = open(path + "/commits_by_author.dat", "w", encoding="utf-8")
 
         lines_by_authors = {}  # cumulated added lines by
         # author. to save memory,
@@ -509,7 +509,7 @@ def create_authors_html(self, data, path):
         domains_by_commits.reverse()  # most first
         f.write('<div class="vtable"><table>')
         f.write("<tr><th>Domains</th><th>Total (%)</th></tr>")
-        fp = open(path + "/domains.dat", "w")
+        fp = open(path + "/domains.dat", "w", encoding="utf-8")
         n = 0
         for domain in domains_by_commits:
             if n == conf["max_domains"]:
@@ -536,7 +536,7 @@ def create_authors_html(self, data, path):
     def create_files_html(self, data, path):
         ###
         # Files
-        f = open(path + "/files.html", "w")
+        f = open(path + "/files.html", "w", encoding="utf-8")
         self.print_header(f)
         f.write("<h1>Files</h1>")
         self.print_nav(f)
@@ -567,7 +567,7 @@ def create_files_html(self, data, path):
                 )
             )
 
-        fg = open(path + "/files_by_date.dat", "w")
+        fg = open(path + "/files_by_date.dat", "w", encoding="utf-8")
         for line in sorted(list(files_by_date)):
             fg.write("%s\n" % line)
         # for stamp in sorted(data.files_by_stamp.keys()):
@@ -610,7 +610,7 @@ def create_files_html(self, data, path):
     def create_lines_html(self, data, path):
         ###
         # Lines
-        f = open(path + "/lines.html", "w")
+        f = open(path + "/lines.html", "w", encoding="utf-8")
         self.print_header(f)
         f.write("<h1>Lines</h1>")
         self.print_nav(f)
@@ -622,7 +622,7 @@ def create_lines_html(self, data, path):
         f.write(html_header(2, "Lines of Code"))
         f.write('<img src="lines_of_code.png" alt="Lines of Code">')
 
-        fg = open(path + "/lines_of_code.dat", "w")
+        fg = open(path + "/lines_of_code.dat", "w", encoding="utf-8")
         for stamp in sorted(data.changes_by_date.keys()):
             fg.write("%d %d\n" % (stamp, data.changes_by_date[stamp]["lines"]))
         fg.close()
@@ -633,7 +633,7 @@ def create_lines_html(self, data, path):
     def create_tags_html(self, data, path):
         ###
         # tags.html
-        f = open(path + "/tags.html", "w")
+        f = open(path + "/tags.html", "w", encoding="utf-8")
         self.print_header(f)
         f.write("<h1>Tags</h1>")
         self.print_nav(f)
@@ -693,7 +693,7 @@ def create_graphs(self, path):
 
     def create_graph_hour_of_day(self, path):
         # hour of day
-        f = open(path + "/hour_of_day.plot", "w")
+        f = open(path + "/hour_of_day.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -711,7 +711,7 @@ def create_graph_hour_of_day(self, path):
 
     def create_graph_day_of_week(self, path):
         # day of week
-        f = open(path + "/day_of_week.plot", "w")
+        f = open(path + "/day_of_week.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -729,7 +729,7 @@ def create_graph_day_of_week(self, path):
 
     def create_graph_domains(self, path):
         # Domains
-        f = open(path + "/domains.plot", "w")
+        f = open(path + "/domains.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -746,7 +746,7 @@ def create_graph_domains(self, path):
 
     def create_graph_month_of_year(self, path):
         # Month of Year
-        f = open(path + "/month_of_year.plot", "w")
+        f = open(path + "/month_of_year.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -764,7 +764,7 @@ def create_graph_month_of_year(self, path):
 
     def create_graph_commits_by_year_month(self, path):
         # commits_by_year_month
-        f = open(path + "/commits_by_year_month.plot", "w")
+        f = open(path + "/commits_by_year_month.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -785,7 +785,7 @@ def create_graph_commits_by_year_month(self, path):
 
     def create_graph_commits_by_year(self, path):
         # commits_by_year
-        f = open(path + "/commits_by_year.plot", "w")
+        f = open(path + "/commits_by_year.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -803,7 +803,7 @@ def create_graph_commits_by_year(self, path):
 
     def create_graph_files_by_date(self, path):
         # Files by date
-        f = open(path + "/files_by_date.plot", "w")
+        f = open(path + "/files_by_date.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -825,7 +825,7 @@ def create_graph_files_by_date(self, path):
 
     def create_graph_lines_of_code(self, path):
         # Lines of Code
-        f = open(path + "/lines_of_code.plot", "w")
+        f = open(path + "/lines_of_code.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -846,7 +846,7 @@ def create_graph_lines_of_code(self, path):
 
     def create_graph_lines_of_code_by_author(self, path):
         # Lines of Code Added per author
-        f = open(path + "/lines_of_code_by_author.plot", "w")
+        f = open(path + "/lines_of_code_by_author.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """
@@ -879,7 +879,7 @@ def create_graph_lines_of_code_by_author(self, path):
 
     def create_graph_commits_by_author(self, path):
         # Commits per author
-        f = open(path + "/commits_by_author.plot", "w")
+        f = open(path + "/commits_by_author.plot", "w", encoding="utf-8")
         f.write(GNUPLOT_COMMON)
         f.write(
             """

gitstats/utils.py

@@ -14,6 +14,22 @@
 conf = load_config()
 
 
+def count_lines_in_text(text):
+    """Cross-platform function to count lines in text"""
+    if not text or not text.strip():
+        return 0
+    return len(text.strip().split("\n"))
+
+
+def filter_lines_by_pattern(text, pattern):
+    """Filter out lines matching a pattern (cross-platform grep -v replacement)"""
+    if not text or not text.strip():
+        return ""
+    lines = text.split("\n")
+    filtered_lines = [line for line in lines if not re.match(pattern, line)]
+    return "\n".join(filtered_lines)
+
+
 def get_version():
     return version("gitstats")
 
@@ -33,21 +49,53 @@ def get_pipe_output(cmds, quiet=False):
     if not quiet and ON_LINUX and os.isatty(1):
         print(">> " + " | ".join(cmds), end=" ")
         sys.stdout.flush()
-    p = subprocess.Popen(cmds[0], stdout=subprocess.PIPE, shell=True)
-    processes = [p]
-    for x in cmds[1:]:
-        p = subprocess.Popen(x, stdin=p.stdout, stdout=subprocess.PIPE, shell=True)
-        processes.append(p)
-    output = p.communicate()[0]
-    for p in processes:
+
+    # Handle cross-platform cases
+    if len(cmds) == 2 and cmds[1] == "wc -l":
+        # Handle line counting cross-platform
+        p = subprocess.Popen(cmds[0], stdout=subprocess.PIPE, shell=True)
+        output = p.communicate()[0]
+        p.wait()
+        try:
+            text = output.decode("utf-8", errors="replace").rstrip("\n")
+        except UnicodeDecodeError:
+            # Fallback for binary files
+            text = output.decode("latin-1", errors="replace").rstrip("\n")
+        line_count = count_lines_in_text(text)
+        result = str(line_count)
+    elif len(cmds) == 2 and cmds[1].startswith("grep -v"):
+        # Handle grep -v cross-platform
+        pattern = cmds[1].split("grep -v ")[1]
+        p = subprocess.Popen(cmds[0], stdout=subprocess.PIPE, shell=True)
+        output = p.communicate()[0]
         p.wait()
+        try:
+            text = output.decode("utf-8", errors="replace").rstrip("\n")
+        except UnicodeDecodeError:
+            text = output.decode("latin-1", errors="replace").rstrip("\n")
+        result = filter_lines_by_pattern(text, pattern)
+    else:
+        # Standard pipe behavior for other cases
+        p = subprocess.Popen(cmds[0], stdout=subprocess.PIPE, shell=True)
+        processes = [p]
+        for x in cmds[1:]:
+            p = subprocess.Popen(x, stdin=p.stdout, stdout=subprocess.PIPE, shell=True)
+            processes.append(p)
+        output = p.communicate()[0]
+        for p in processes:
+            p.wait()
+        try:
+            result = output.decode("utf-8", errors="replace").rstrip("\n")
+        except UnicodeDecodeError:
+            result = output.decode("latin-1", errors="replace").rstrip("\n")
+
     end = time.time()
     if not quiet:
         if ON_LINUX and os.isatty(1):
             print("\r", end=" ")
         print("[%.5f] >> %s" % (end - start, " | ".join(cmds)))
     exectime_external += end - start
-    return output.decode("utf-8").rstrip("\n")
+    return result
 
 
 def get_commit_range(defaultrange="HEAD", end_only=False):