Skip to content

Commit 468befc

Browse files
nil0x42nil0x42
committed
doc: Add gif demo to showcase progressbar
1 parent 1eb1da2 commit 468befc

File tree

4 files changed

+291
-13
lines changed

4 files changed

+291
-13
lines changed

‎.github/images/1-comparison-v2.png‎

48.8 KB
Loading

‎.github/images/demo.gif‎

996 KB
Loading

‎.github/images/demo.gif-terminalizer.yml‎

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
# The configurations that used for the recording, feel free to edit them
2+
config:
3+
4+
# Specify a command to be executed
5+
# like `/bin/bash -l`, `ls`, or any other commands
6+
# the default is bash for Linux
7+
# or powershell.exe for Windows
8+
command: /bin/bash -l
9+
10+
# Specify the current working directory path
11+
# the default is the current working directory path
12+
cwd: /tmp/uu
13+
14+
# Export additional ENV variables
15+
env:
16+
recording: true
17+
18+
# Explicitly set the number of columns
19+
# or use `auto` to take the current
20+
# number of columns of your shell
21+
cols: 81
22+
23+
# Explicitly set the number of rows
24+
# or use `auto` to take the current
25+
# number of rows of your shell
26+
rows: 17
27+
28+
# Amount of times to repeat GIF
29+
# If value is -1, play once
30+
# If value is 0, loop indefinitely
31+
# If value is a positive number, loop n times
32+
repeat: 0
33+
34+
# Quality
35+
# 1 - 100
36+
quality: 100
37+
38+
# Delay between frames in ms
39+
# If the value is `auto` use the actual recording delays
40+
frameDelay: auto
41+
42+
# Maximum delay between frames in ms
43+
# Ignored if the `frameDelay` isn't set to `auto`
44+
# Set to `auto` to prevent limiting the max idle time
45+
maxIdleTime: 10000
46+
47+
# The surrounding frame box
48+
# The `type` can be null, window, floating, or solid`
49+
# To hide the title use the value null
50+
# Don't forget to add a backgroundColor style with a null as type
51+
frameBox:
52+
type: floating
53+
title: github.com/nil0x42/duplicut
54+
style:
55+
backgroundColor: black
56+
border: 0px black solid
57+
# boxShadow: none
58+
# margin: 0px
59+
60+
# Add a watermark image to the rendered gif
61+
# You need to specify an absolute path for
62+
# the image on your machine or a URL, and you can also
63+
# add your own CSS styles
64+
watermark:
65+
imagePath: null
66+
style:
67+
position: absolute
68+
right: 15px
69+
bottom: 15px
70+
width: 100px
71+
opacity: 0.9
72+
73+
# Cursor style can be one of
74+
# `block`, `underline`, or `bar`
75+
cursorStyle: block
76+
77+
# Font family
78+
# You can use any font that is installed on your machine
79+
# in CSS-like syntax
80+
fontFamily: "Monaco, Lucida Console, Ubuntu Mono, Monospace"
81+
82+
# The size of the font
83+
fontSize: 18
84+
85+
# The height of lines
86+
lineHeight: 1
87+
88+
# The spacing between letters
89+
letterSpacing: 0
90+
91+
# Theme
92+
# theme:
93+
# background: "transparent"
94+
# foreground: "#afafaf"
95+
# cursor: "#c7c7c7"
96+
# black: "#232628"
97+
# red: "#fc4384"
98+
# green: "#b3e33b"
99+
# yellow: "#ffa727"
100+
# blue: "#75dff2"
101+
# magenta: "#ae89fe"
102+
# cyan: "#708387"
103+
# white: "#d5d5d0"
104+
# brightBlack: "#626566"
105+
# brightRed: "#ff7fac"
106+
# brightGreen: "#c8ed71"
107+
# brightYellow: "#ebdf86"
108+
# brightBlue: "#75dff2"
109+
# brightMagenta: "#ae89fe"
110+
# brightCyan: "#b1c6ca"
111+
# brightWhite: "#f9f9f4"
112+
113+
theme:
114+
# background & texte
115+
background: "transparent"
116+
foreground: "#afafaf" # gris clair
117+
cursor: "#FFFFFF" # blanc
118+
119+
# Couleurs standard
120+
black: "#000000"
121+
red: "#AA0000"
122+
green: "#00AA00"
123+
yellow: "#AA5500"
124+
blue: "#0000AA"
125+
magenta: "#AA00AA"
126+
cyan: "#00AAAA"
127+
white: "#AAAAAA"
128+
129+
# Couleurs vives (bright)
130+
brightBlack: "#444444"
131+
brightRed: "#FF4444"
132+
brightGreen: "#44FF44"
133+
brightYellow: "#FFFF44"
134+
brightBlue: "#4444FF"
135+
brightMagenta: "#FF44FF"
136+
brightCyan: "#44FFFF"
137+
brightWhite: "#FFFFFF"
138+
139+
# Records, feel free to edit them
140+
records:
141+
- delay: 3
142+
content: "\e[?2004h\r\e[35mduplicut@localhost $ \e[0m"
143+
- delay: 139
144+
content: d
145+
- delay: 113
146+
content: u
147+
- delay: 123
148+
content: p
149+
- delay: 44
150+
content: l
151+
- delay: 75
152+
content: i
153+
- delay: 120
154+
content: c
155+
- delay: 123
156+
content: u
157+
- delay: 100
158+
content: t
159+
- delay: 114
160+
content: ' '
161+
- delay: 109
162+
content: '-'
163+
- delay: 213
164+
content: o
165+
- delay: 140
166+
content: ' '
167+
- delay: 101
168+
content: d
169+
- delay: 136
170+
content: e
171+
- delay: 139
172+
content: d
173+
- delay: 50
174+
content: u
175+
- delay: 106
176+
content: p
177+
- delay: 82
178+
content: e
179+
- delay: 140
180+
content: .
181+
- delay: 94
182+
content: t
183+
- delay: 108
184+
content: x
185+
- delay: 101
186+
content: t
187+
- delay: 102
188+
content: ' '
189+
- delay: 101
190+
content: w
191+
- delay: 87
192+
content: o
193+
- delay: 102
194+
content: r
195+
- delay: 100
196+
content: d
197+
- delay: 108
198+
content: 'list.txt '
199+
- delay: 64
200+
content: "\r\n\e[?2004l\r"
201+
- delay: 770
202+
content: "\r\e[2K\e[0mtime: \e[1m1s \e[1;33m12.4%\e[0m (ETA: \e[1m 9s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
203+
- delay: 253
204+
content: "\r\e[2K\e[0mtime: \e[1m1s \e[1;33m18.2%\e[0m (ETA: \e[1m 9s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
205+
- delay: 254
206+
content: "\r\e[2K\e[0mtime: \e[1m1s \e[1;33m25.1%\e[0m (ETA: \e[1m 9s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
207+
- delay: 253
208+
content: "\r\e[2K\e[0mtime: \e[1m1s \e[1;33m30.9%\e[0m (ETA: \e[1m 9s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
209+
- delay: 253
210+
content: "\r\e[2K\e[0mtime: \e[1m2s \e[1;33m34.6%\e[0m (ETA: \e[1m 6s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
211+
- delay: 253
212+
content: "\r\e[2K\e[0mtime: \e[1m2s \e[1;33m40.5%\e[0m (ETA: \e[1m 6s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
213+
- delay: 253
214+
content: "\r\e[2K\e[0mtime: \e[1m2s \e[1;33m45.1%\e[0m (ETA: \e[1m 6s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
215+
- delay: 254
216+
content: "\r\e[2K\e[0mtime: \e[1m2s \e[1;33m50.0%\e[0m (ETA: \e[1m 6s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
217+
- delay: 254
218+
content: "\r\e[2K\e[0mtime: \e[1m3s \e[1;33m54.6%\e[0m (ETA: \e[1m 4s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
219+
- delay: 253
220+
content: "\r\e[2K\e[0mtime: \e[1m3s \e[1;33m58.7%\e[0m (ETA: \e[1m 4s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
221+
- delay: 254
222+
content: "\r\e[2K\e[0mtime: \e[1m3s \e[1;33m62.7%\e[0m (ETA: \e[1m 4s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
223+
- delay: 254
224+
content: "\r\e[2K\e[0mtime: \e[1m3s \e[1;33m74.9%\e[0m (ETA: \e[1m 4s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
225+
- delay: 254
226+
content: "\r\e[2K\e[0mtime: \e[1m4s \e[1;33m86.3%\e[0m (ETA: \e[1m 0s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
227+
- delay: 253
228+
content: "\r\e[2K\e[0mtime: \e[1m4s \e[1;33m90.2%\e[0m (ETA: \e[1m 0s\e[0m) \e[36m[2/3] \e[1mdedupe chunk 1/1 \e[0;2;36m(task 1/1)\e[0;36m ...\e[0m"
229+
- delay: 253
230+
content: "\r\e[2K\e[0mtime: \e[1m4s \e[1;33m94.7%\e[0m (ETA: \e[1m 0s\e[0m) \e[36m[3/3] \e[1mremoving tagged lines\e[0;36m ...\e[0m"
231+
- delay: 254
232+
content: "\r\e[2K\e[0mtime: \e[1m4s \e[1;33m97.4%\e[0m (ETA: \e[1m 0s\e[0m) \e[36m[3/3] \e[1mremoving tagged lines\e[0;36m ...\e[0m"
233+
- delay: 182
234+
content: "\r\e[2K\e[0mtime: \e[1m5s \e[1;33m100.0%\e[0m (ETA: \e[1m 0s\e[0m) \e[36m[3/3] \e[1mremoving tagged lines\e[0;36m ...\e[0m\r\nduplicut successfully removed \e[1m20107865\e[0m duplicates and \e[1m12548\e[0m filtered lines in \e[1m5s\e[0m\r\n\e[?2004h\r\e[35mduplicut@localhost $ \e[0m"
235+
- delay: 943
236+
content: l
237+
- delay: 114
238+
content: l
239+
- delay: 269
240+
content: " * | grep -E '[0-9]+M'\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b"
241+
- delay: 159
242+
content: "\r\n\e[?2004l\r"
243+
- delay: 12
244+
content: "-rw-r--r-- 1 user user \e[01;31m\e[K438M\e[m\e[K May 15 16:22 dedupe.txt\r\n-rw-r--r-- 1 user user \e[01;31m\e[K622M\e[m\e[K May 12 16:53 wordlist.txt\r\n\e[?2004h\r\e[35mduplicut@localhost $ \e[0m"
245+
- delay: 860
246+
content: h
247+
- delay: 132
248+
content: e
249+
- delay: 50
250+
content: a
251+
- delay: 133
252+
content: d
253+
- delay: 213
254+
content: " -n3 *\b\b\b\b\b\b"
255+
- delay: 265
256+
content: "\r\n\e[?2004l\r"
257+
- delay: 10
258+
content: "==> dedupe.txt <==\r\n12345\r\nabc123\r\npassword\r\n\r\n==> wordlist.txt <==\r\n12345\r\nabc123\r\npassword\r\n\e[?2004h\r\e[35mduplicut@localhost $ \e[0m"
259+
- delay: 6000
260+
content: "\r"

‎README.md‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@
3333
<a href="https://www.blackarch.org/misc.html" target="_blank">
3434
<img src="https://img.shields.io/static/v1?label=BlackArch&message=packaged&color=red&logo=archlinux&logoColor=006">
3535
</a>
36-
<a href="https://twitter.com/intent/follow?screen_name=nil0x42" target="_blank">
37-
<img src="https://img.shields.io/twitter/follow/nil0x42.svg?logo=twitter" akt="follow on twitter">
36+
</a>
37+
<a href="https://x.com/intent/follow?screen_name=nil0x42" target="_blank">
38+
<img src="https://img.shields.io/badge/Follow-@nil0x42-6aa?logo=x" akt="Follow on X">
3839
</a>
3940
</p>
4041

@@ -50,6 +51,8 @@
5051

5152
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
5253

54+
<a href="#"><img align="right" src=".github/images/1-comparison-v2.png" width="55%" /></a>
55+
5356
### :book: Overview
5457

5558
Nowadays, password wordlist creation usually implies concatenating
@@ -58,42 +61,58 @@ multiple data sources.
5861
Ideally, most probable passwords should stand at start of the wordlist,
5962
so most common passwords are cracked instantly.
6063

64+
6165
With existing *dedupe tools* you are forced to choose
6266
if you prefer to *preserve the order **OR** handle massive wordlists*.
6367

64-
Unfortunately, **wordlist creation requires both**:
68+
Unfortunately, **wordlist creation requires both**
6569

66-
![][img-1-comparison]
6770

6871
> **So i wrote duplicut in [highly optimized C][get-next-line] to address this very specific need :nerd\_face: :computer:**
6972
73+
<a href="#"><img align="center" src=".github/images/demo.gif" width="80%" /></a>
74+
<br>
75+
7076
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
7177

7278
### :bulb: Quick start
7379

7480
```sh
75-
git clone https://github.com/nil0x42/duplicut
76-
cd duplicut/ && make
77-
./duplicut wordlist.txt -o clean-wordlist.txt
81+
git clone https://github.com/nil0x42/duplicut # download ...
82+
cd duplicut/ && make # compile ...
83+
./duplicut wordlist.txt -o clean-wordlist.txt # dedupe !!!
7884
```
7985

8086
### :wrench: Options
8187

82-
![][img-4-help]
88+
```
89+
-o, --outfile <FILE> Write result to <FILE>
90+
-t, --threads <NUM> Max threads to use (default max)
91+
-m, --memlimit <VALUE> Limit max used memory (default max)
92+
-l, --line-max-size <NUM> Max line size (default 1024, max 4095)
93+
-p, --printable Filter ascii printable lines
94+
-c, --lowercase Convert wordlist to lowercase
95+
-C, --uppercase Convert wordlist to uppercase
96+
-D, --dupfile <FILE> Write dupes to <FILE> (slows down duplicut)
97+
-h, --help Display this help and exit
98+
-v, --version Output version information and exit
99+
```
83100

84101
* **Features**:
85102
* Handle massive wordlists, even those whose size exceeds available RAM
86-
* Filter lines by max length (`-l` option)
87-
* Can remove lines containing non-printable ASCII chars (`-p` option)
88-
* Press any key to show program status at runtime.
103+
* Filter lines by max length (`--line-max-size` option)
104+
* Filter-out lines with non-printable ASCII chars (`--printable` option)
105+
* Save found duplicates to another file (`--dupfile` option)
106+
* Ignore case with `--lowercase` or `--uppercase` options
107+
* Displays a nice progress bar with ETA
89108

90109
* **Implementation**:
91110
* Written in pure C code, designed to be fast
92111
* Compressed hashmap items on 64 bit platforms
93112
* Multithreading support
94113

95114
* **Limitations**:
96-
* Any line longer than 4095 chars is ignored
115+
* `--line-max-size` can't exceed 4095
97116

98117
### :book: Technical Details
99118

@@ -130,7 +149,6 @@ make debug level=1
130149

131150
[get-next-line]: https://github.com/nil0x42/duplicut/blob/master/src/line.c#L39
132151

133-
[img-1-comparison]: .github/images/1-comparison.png
134152
[img-2-line-struct]: .github/images/2-line-struct.png
135153
[img-3-chunked-processing]: .github/images/3-chunked-processing.png
136154
[img-4-help]: .github/images/4-help.png

0 commit comments

Comments
 (0)