Remove duplicate and consecutive duplicate lines

Closes #893

Author: dail8859

src/NotepadNext/ByteArrayUtils.h

@@ -0,0 +1,119 @@
+/*
+ * This file is part of Notepad Next.
+ * Copyright 2025 Justin Dailey
+ *
+ * Notepad Next is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Notepad Next is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Notepad Next.  If not, see <https://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include <QByteArray>
+#include <QByteArrayView>
+#include <QList>
+#include <QSet>
+#include <cstring>
+#include <unordered_set>
+
+namespace ByteArrayUtils
+{
+
+// Zero-copy split: returns views into 'source'.
+// The source QByteArray must remain alive while using the views.
+inline QList<QByteArrayView> split(const QByteArray& source, const QByteArray& delimiter)
+{
+    QList<QByteArrayView> out;
+
+    const char* base = source.constData();
+    qsizetype len = source.size();
+    qsizetype dlen = delimiter.size();
+
+    qsizetype start = 0;
+
+    while (true) {
+        qsizetype pos = source.indexOf(delimiter, start);
+        if (pos < 0) {
+            out.append(QByteArrayView(base + start, len - start));
+            break;
+        }
+
+        out.append(QByteArrayView(base + start, pos - start));
+        start = pos + dlen;
+    }
+
+    return out;
+}
+
+// Remove duplicates and preserve order.
+inline void removeDuplicates(QList<QByteArrayView>& parts)
+{
+    std::unordered_set<QByteArray, std::hash<QByteArray>> seen;
+
+    auto newEnd = std::remove_if(parts.begin(), parts.end(), [&seen](const QByteArrayView& v) {
+        QByteArray key(v.data(), v.size()); // copy once per unique value
+        if (seen.find(key) != seen.end())
+            return true; // duplicate, remove
+        seen.insert(std::move(key));
+        return false;    // first occurrence, keep
+    });
+
+    parts.erase(newEnd, parts.end());
+}
+
+
+// Remove only *consecutive* duplicates.
+// Example: A A B B B C A  →  A B C A
+inline void removeConsecutiveDuplicates(QList<QByteArrayView>& parts)
+{
+    auto newEnd = std::unique(parts.begin(), parts.end(), [](const QByteArrayView& a, const QByteArrayView& b) {
+        return a == b;
+    });
+
+    parts.erase(newEnd, parts.end());
+}
+
+
+inline QByteArray join(const QList<QByteArrayView>& parts, const QByteArray& delimiter)
+{
+    if (parts.isEmpty())
+        return QByteArray();
+
+    const qsizetype dlen = delimiter.size();
+
+    // Compute total size
+    qsizetype total = 0;
+    for (const auto& v : parts)
+        total += v.size();
+    total += dlen * (parts.size() - 1);
+
+    // Allocate final QByteArray
+    QByteArray out;
+    out.resize(total);
+
+    // Copy segments
+    char* dst = out.data();
+    for (int i = 0; i < parts.size(); ++i) {
+        const auto& v = parts[i];
+
+        memcpy(dst, v.data(), v.size());
+        dst += v.size();
+
+        if (i + 1 < parts.size()) {
+            memcpy(dst, delimiter.constData(), dlen);
+            dst += dlen;
+        }
+    }
+
+    return out;
+}
+
+} // namespace ByteArrayViewUtils

src/NotepadNext/NotepadNext.pro

@@ -142,6 +142,7 @@ SOURCES += \
 HEADERS += \
     ActionUtils.h \
     ApplicationSettings.h \
+    ByteArrayUtils.h \
     ColorPickerDelegate.h \
     ComboBoxDelegate.h \
     Converter.h \

src/NotepadNext/ScintillaNext.cpp

@@ -18,8 +18,10 @@
 
 
 #include "ScintillaNext.h"
+#include "Finder.h"
 #include "ScintillaCommenter.h"
 
+#include "ByteArrayUtils.h"
 #include "uchardet.h"
 #include <cinttypes>
 
@@ -498,6 +500,45 @@ void ScintillaNext::uncommentLineSelection()
     sc.uncommentSelection();
 }
 
+void ScintillaNext::removeDuplicateLines()
+{
+    QByteArray data = QByteArray::fromRawData((char*) characterPointer(), textLength());
+    const QByteArray delim = eolString();
+
+    auto lines = ByteArrayUtils::split(data, delim);
+    int originalLineCount = lines.length();
+    ByteArrayUtils::removeDuplicates(lines);
+
+    if (originalLineCount == lines.length()){
+        return; // No lines were removed
+    }
+
+    QByteArray result = ByteArrayUtils::join(lines, delim);
+
+    const UndoAction ua(this);
+    setTargetRange(0, textLength());
+    replaceTarget(result.length(), result.constData());
+}
+
+void ScintillaNext::removeConsecutiveDuplicateLines()
+{
+    QByteArray data = QByteArray::fromRawData((char*) characterPointer(), textLength());
+    const QByteArray delim = eolString();
+
+    auto lines = ByteArrayUtils::split(data, delim);
+    int originalLineCount = lines.length();
+    ByteArrayUtils::removeConsecutiveDuplicates(lines);
+    QByteArray result = ByteArrayUtils::join(lines, delim);
+
+    if (originalLineCount == lines.length()){
+        return; // No lines were removed
+    }
+
+    const UndoAction ua(this);
+    setTargetRange(0, textLength());
+    replaceTarget(result.length(), result.constData());
+}
+
 void ScintillaNext::dragEnterEvent(QDragEnterEvent *event)
 {
     // Ignore all drag and drop events with urls and let the main application handle it

src/NotepadNext/ScintillaNext.h

@@ -133,6 +133,9 @@ public slots:
     void commentLineSelection();
     void uncommentLineSelection();
 
+    void removeDuplicateLines();
+    void removeConsecutiveDuplicateLines();
+
 signals:
     void aboutToSave();
     void saved();

src/NotepadNext/dialogs/MainWindow.cpp

@@ -250,6 +250,12 @@ MainWindow::MainWindow(NotepadNextApplication *app) :
         // Regex will also not delete the final blank line
         editor->deleteTrailingEmptyLines();
     });
+    connect(ui->actionRemoveDuplicateLines, &QAction::triggered, this, [=]() {
+        currentEditor()->removeDuplicateLines();
+    });
+    connect(ui->actionRemoveConsecutiveDuplicateLines, &QAction::triggered, this, [=]() {
+        currentEditor()->removeConsecutiveDuplicateLines();
+    });
 
     connect(ui->actionColumnMode, &QAction::triggered, this, [=]() {
         ColumnEditorDialog *columnEditor = findChild<ColumnEditorDialog *>(QString(), Qt::FindDirectChildrenOnly);

src/NotepadNext/dialogs/MainWindow.ui

@@ -137,6 +137,8 @@
      <addaction name="actionMoveSelectedLinesDown"/>
      <addaction name="separator"/>
      <addaction name="actionRemoveEmptyLines"/>
+     <addaction name="actionRemoveDuplicateLines"/>
+     <addaction name="actionRemoveConsecutiveDuplicateLines"/>
     </widget>
     <widget class="QMenu" name="menuCommentUncomment">
      <property name="title">
@@ -1472,6 +1474,16 @@
     <string>Clear All Styles</string>
    </property>
   </action>
+  <action name="actionRemoveDuplicateLines">
+   <property name="text">
+    <string>Remove Duplicate Lines</string>
+   </property>
+  </action>
+  <action name="actionRemoveConsecutiveDuplicateLines">
+   <property name="text">
+    <string>Remove Consecutive Duplicate Lines</string>
+   </property>
+  </action>
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <customwidgets>