Skip to content

Commit cf40fd2

Browse files
bit4woobit4woo
committed
get title多线程逻辑优化
1 parent e3ad9a3 commit cf40fd2

File tree

6 files changed

+152
-170
lines changed

6 files changed

+152
-170
lines changed

‎src/thread/Producer.java‎

Lines changed: 51 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.util.Map;
99
import java.util.Set;
1010
import java.util.concurrent.BlockingQueue;
11+
import java.util.concurrent.TimeUnit;
1112

1213
import com.bit4woo.utilbox.utils.DomainUtils;
1314

@@ -20,87 +21,71 @@
2021
import config.ConfigName;
2122
import title.LineEntry;
2223

23-
/**
24-
* @author bit4woo
25-
* @github https://github.com/bit4woo
26-
* @version CreateTime：Jun 25, 2020 2:35:31 PM
27-
*/
28-
2924
/**
3025
* 执行web请求，获取title的线程
31-
*
32-
* @author bit4woo
3326
*
3427
*/
3528

36-
public class Producer extends Thread {//Producer do
37-
private final BlockingQueue<Map.Entry<String,String>> domainQueue;//use to store domains
38-
private volatile boolean stopflag = false;
29+
public class Producer extends Thread {// Producer do
30+
private final BlockingQueue<Map.Entry<String, String>> domainQueue;// use to store domains
3931

40-
private static IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks();//静态变量，burp插件的逻辑中，是可以保证它被初始化的。;
32+
private static IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks();// 静态变量，burp插件的逻辑中，是可以保证它被初始化的。;
4133
public PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true);
4234
public PrintWriter stderr = new PrintWriter(callbacks.getStderr(), true);
4335
public IExtensionHelpers helpers = callbacks.getHelpers();
4436
private GUIMain guiMain;
4537

46-
public Producer(GUIMain guiMain,BlockingQueue<Map.Entry<String,String>> domainQueue,int threadNo) {
38+
public Producer(GUIMain guiMain, BlockingQueue<Map.Entry<String, String>> domainQueue, int threadNo) {
4739
this.guiMain = guiMain;
4840
this.domainQueue = domainQueue;
49-
stopflag= false;
50-
this.setName(this.getClass().getName()+threadNo);
51-
}
52-
53-
public void setStopflag(boolean stop) {
54-
stopflag = stop;
41+
this.setName(this.getClass().getName() + threadNo);
5542
}
5643

5744
@Override
5845
public void run() {
59-
while(true){
60-
try {
61-
if (domainQueue.isEmpty() ) {
62-
stdout.println(getName()+" exited. due to domainQueue is empty");
46+
try {
47+
while (true) {
48+
if (Thread.currentThread().isInterrupted()) {
49+
stdout.println(getName() + " interrupted");
6350
break;
6451
}
65-
if (Thread.interrupted()){//没有起作用！
66-
stdout.println(getName()+" exited. due to thread interrupt signal received");
67-
break;
68-
}
69-
if (stopflag){
70-
stdout.println(getName()+" exited. due to stop flag is true");
52+
53+
Map.Entry<String, String> entry = domainQueue.poll(1, TimeUnit.SECONDS);
54+
if (entry == null) {
55+
stdout.println(getName() + " queue empty, exit");
7156
break;
7257
}
7358

74-
Map.Entry<String,String> entry = domainQueue.take();
7559
String host = entry.getKey();
7660
String type = entry.getValue();
77-
Set<URL> urls = new HashSet<>(new DomainUtils().toURLs(host));
61+
Set<URL> urls = new HashSet<>(DomainUtils.toURLs(host));
7862

7963
List<LineEntry> tempEntries = new ArrayList<LineEntry>();
80-
for (URL Url:urls) {
64+
for (URL Url : urls) {
8165
LineEntry item = new LineEntry(Url).firstRequest(guiMain.getTitlePanel().getTempConfig());
8266

8367
ConfigManager.doFilter(item);
84-
if (ConfigManager.getBooleanConfigByKey(ConfigName.removeItemIfIgnored) && item.getCheckStatus().equals(LineEntry.CheckStatus_Checked)) {
68+
if (ConfigManager.getBooleanConfigByKey(ConfigName.removeItemIfIgnored)
69+
&& item.getCheckStatus().equals(LineEntry.CheckStatus_Checked)) {
8570
continue;
8671
}
8772
item.setEntrySource(type);
8873

8974
String url = item.getUrl();
90-
if (item.getEntryType().equals(LineEntry.EntryType_Web)){
75+
if (item.getEntryType().equals(LineEntry.EntryType_Web)) {
9176
LineEntry linefound = findHistory(url.toString());
9277
if (null != linefound) {
9378
linefound.getEntryTags().remove(LineEntry.Tag_NotTargetBaseOnCertInfo);
9479
linefound.getEntryTags().remove(LineEntry.Tag_NotTargetBaseOnBlackList);
9580
item.getComments().addAll(linefound.getComments());
9681
item.setAssetType(linefound.getAssetType());
9782
try {
98-
//长度的判断不准确，不再使用，就记录以前的状态！时间就记录上次完成渗透的时间
83+
// 长度的判断不准确，不再使用，就记录以前的状态！时间就记录上次完成渗透的时间
9984
if (url.equalsIgnoreCase(linefound.getUrl())) {
10085
item.setCheckStatus(linefound.getCheckStatus());
10186
item.setTime(linefound.getTime());
10287
}
103-
}catch(Exception err) {
88+
} catch (Exception err) {
10489
err.printStackTrace(stderr);
10590
}
10691
}
@@ -110,61 +95,65 @@ public void run() {
11095

11196
tempEntries = ConfigManager.doSameHostFilter(tempEntries);
11297

113-
for (LineEntry item:tempEntries) {
114-
if (ConfigManager.getBooleanConfigByKey(ConfigName.removeItemIfIgnored) && item.getCheckStatus().equals(LineEntry.CheckStatus_Checked)) {
98+
for (LineEntry item : tempEntries) {
99+
if (ConfigManager.getBooleanConfigByKey(ConfigName.removeItemIfIgnored)
100+
&& item.getCheckStatus().equals(LineEntry.CheckStatus_Checked)) {
115101
continue;
116102
}
117103
guiMain.getTitlePanel().getTitleTable().getLineTableModel().addNewLineEntry(item);
118-
//stdout.println(new LineEntry(messageinfo,true).ToJson());
104+
// stdout.println(new LineEntry(messageinfo,true).ToJson());
119105
int leftTaskNum = domainQueue.size();
120-
stdout.println(String.format("+++ [%s] +++ get title done %s tasks left",item.getUrl(),leftTaskNum));
106+
stdout.println(
107+
String.format("+++ [%s] +++ get title done %s tasks left", item.getUrl(), leftTaskNum));
121108
}
122-
123-
} catch (Exception error) {
124-
error.printStackTrace(stderr);
125-
continue;//unnecessary
126-
//java.lang.RuntimeException can't been catched, why?
127109
}
110+
} catch (InterruptedException ex) {
111+
Thread.currentThread().interrupt();
112+
stdout.println(getName() + " interrupted during execution");
113+
} catch (Exception error) {
114+
error.printStackTrace(stderr);
128115
}
116+
129117
}
130118

131119
public LineEntry findHistory(String url) {
132-
IndexedHashMap<String,LineEntry> HistoryLines = guiMain.getTitlePanel().getBackupLineEntries();
133-
if (HistoryLines == null) return null;
120+
IndexedHashMap<String, LineEntry> HistoryLines = guiMain.getTitlePanel().getBackupLineEntries();
121+
if (HistoryLines == null)
122+
return null;
134123
LineEntry found = HistoryLines.get(url);
135124
if (found != null) {
136-
//HistoryLines.remove(url,null);
137-
//当对象是HashMap时不使用remove和put操作，避免ConcurrentModificationException问题，因为这2个操作都会让map的长度发生变化，从而导致问题
138-
//但是当线程对象是ConcurrentHashMap时，可以直接remove。
139-
//但是为了效率考虑不进行删除操，以前为什么要替换成null？？？忘记了
125+
// HistoryLines.remove(url,null);
126+
// 当对象是HashMap时不使用remove和put操作，避免ConcurrentModificationException问题，因为这2个操作都会让map的长度发生变化，从而导致问题
127+
// 但是当线程对象是ConcurrentHashMap时，可以直接remove。
128+
// 但是为了效率考虑不进行删除操，以前为什么要替换成null？？？忘记了
140129
return found;
141130
}
142131

143-
//根据host进行查找的逻辑，不会导致手动保存的条目被替换为null，因为手动保存的条目IP列表为空
144-
for (LineEntry line:HistoryLines.values()) {
145-
if (line== null) {
132+
// 根据host进行查找的逻辑，不会导致手动保存的条目被替换为null，因为手动保存的条目IP列表为空
133+
for (LineEntry line : HistoryLines.values()) {
134+
if (line == null) {
146135
continue;
147136
}
148-
try{//根据host查找
149-
String host = new URL(url).getHost();//可能是域名、也可能是IP
137+
try {// 根据host查找
138+
String host = new URL(url).getHost();// 可能是域名、也可能是IP
150139

151-
Set<String> lineHost = new HashSet<>(line.getIPSet());//解析得到的IP集合
140+
Set<String> lineHost = new HashSet<>(line.getIPSet());// 解析得到的IP集合
152141
lineHost.add(line.getHost());
153142
if (lineHost.contains(host)) {
154-
//HistoryLines.remove(line.getUrl());//如果有相同URL的记录，就删除这个记录。//ConcurrentModificationException
155-
//HistoryLines.replace(url,null);
143+
// HistoryLines.remove(line.getUrl());//如果有相同URL的记录，就删除这个记录。//ConcurrentModificationException
144+
// HistoryLines.replace(url,null);
156145
return line;
157146
}
158-
}catch (Exception e){
147+
} catch (Exception e) {
159148
e.printStackTrace(BurpExtender.getStderr());
160149
}
161150
}
162151
return null;
163152
}
164153

165154
public static void main(String[] args) {
166-
int i= 0;
167-
while(true) {
155+
int i = 0;
156+
while (true) {
168157
if (i >= 10) {
169158
System.out.println("exited.");
170159
break;

‎src/thread/ThreadGetTitle.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package thread;
2+
3+
import java.io.PrintWriter;
4+
import java.util.ArrayList;
5+
import java.util.Collections;
6+
import java.util.HashMap;
7+
import java.util.List;
8+
import java.util.Map;
9+
import java.util.concurrent.BlockingQueue;
10+
import java.util.concurrent.LinkedBlockingQueue;
11+
12+
import GUI.GUIMain;
13+
import burp.BurpExtender;
14+
import burp.IBurpExtenderCallbacks;
15+
import burp.IExtensionHelpers;
16+
17+
public class ThreadGetTitle extends Thread{
18+
private HashMap<String,String> domains;
19+
private final List<Producer> plist = Collections.synchronizedList(new ArrayList<>());
20+
21+
22+
private static IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks();//静态变量，burp插件的逻辑中，是可以保证它被初始化的。;
23+
public PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true);
24+
public PrintWriter stderr = new PrintWriter(callbacks.getStderr(), true);
25+
public IExtensionHelpers helpers = callbacks.getHelpers();
26+
private int threadNumber;
27+
private GUIMain guiMain;
28+
29+
public ThreadGetTitle(GUIMain guiMain,HashMap<String,String> domains,int threadNumber) {
30+
this.guiMain = guiMain;
31+
this.domains = domains;
32+
this.threadNumber = threadNumber;
33+
}
34+
35+
@Override
36+
public void run(){
37+
stdout.println(String.format("~~~~~~~~~~~~~use %s threads~~~~~~~~~~~~~",threadNumber));
38+
stdout.println("~~~~~~~~~~~~~Start threading Get Title~~~~~~~~~~~~~ total task number: "+domains.size());
39+
BlockingQueue<Map.Entry<String,String>> domainQueue = new LinkedBlockingQueue<>();//use to store domains
40+
domainQueue.addAll(domains.entrySet());
41+
42+
for (int i=0;i<threadNumber;i++) {
43+
Producer p = new Producer(guiMain,domainQueue,i);
44+
//p.setDaemon(true);//将子线程设置为守护线程，会随着主线程的结束而立即结束。"主线程的结束"指的是”只有当JVM也退出时才可以！”
45+
plist.add(p);
46+
p.start();
47+
}
48+
49+
for (Producer p : plist) {
50+
try {
51+
p.join(1000);
52+
} catch (InterruptedException e) {
53+
Thread.currentThread().interrupt();
54+
break;
55+
}
56+
}
57+
58+
stdout.println(getName()+" finished");
59+
}
60+
61+
public void stopAll() {
62+
if (plist == null) return;
63+
64+
for (Producer p : plist) {
65+
p.interrupt();//必须配合Thread.currentThread().isInterrupted()逻辑，否则不起作用
66+
}
67+
68+
for (Producer p : plist) {
69+
try {
70+
p.join(1000);
71+
} catch (InterruptedException e) {
72+
Thread.currentThread().interrupt();
73+
}
74+
}
75+
stdout.println("~~~~~~~~~~~~~all sub-threads exit!~~~~~~~~~~~~~");
76+
}
77+
}

‎src/thread/ThreadGetTitleWithForceStop.java‎

Lines changed: 0 additions & 84 deletions
This file was deleted.

‎src/title/GetTitleMenu.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -169,14 +169,14 @@ protected void done() {
169169
});
170170

171171

172-
StopItem = new JMenuItem(new AbstractAction("Force Stop Get Title Threads") {
172+
StopItem = new JMenuItem(new AbstractAction("Stop All Get Title Threads") {
173173
@Override
174174
public void actionPerformed(ActionEvent actionEvent) {
175175
if (guiMain.getTitlePanel().getThreadGetTitle() != null &&
176176
guiMain.getTitlePanel().getThreadGetTitle().isAlive() ){
177-
int result = JOptionPane.showConfirmDialog(null,"Are You Sure To [Force Stop] All Get Title Threads ?");
177+
int result = JOptionPane.showConfirmDialog(null,"Are You Sure To [Stop] All Get Title Threads ?");
178178
if (result == JOptionPane.YES_OPTION){
179-
guiMain.getTitlePanel().getThreadGetTitle().forceStopThreads();
179+
guiMain.getTitlePanel().getThreadGetTitle().stopAll();
180180
}
181181
}
182182
}

0 commit comments

Comments
 (0)