Skip to content

Commit 091d4f9

Browse files
thc202thc202
authored
Merge pull request #5712 from psiinon/oast/days
OAST Trim database
2 parents 42fedc8 + cc22163 commit 091d4f9

File tree

13 files changed

+257
-16
lines changed

13 files changed

+257
-16
lines changed

‎addOns/database/src/main/java/org/zaproxy/addon/database/Database.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import javax.jdo.JDOHelper;
2828
import javax.jdo.PersistenceManager;
2929
import javax.jdo.PersistenceManagerFactory;
30+
import javax.jdo.Query;
3031
import javax.jdo.Transaction;
3132
import org.datanucleus.PropertyNames;
3233
import org.datanucleus.api.jdo.JDOPersistenceManagerFactory;
@@ -65,6 +66,9 @@ protected Database(String persistenceUnitName, ClassLoader classLoader) {
6566
jdoProperties.put(PropertyNames.PROPERTY_SCHEMA_VALIDATE_COLUMNS, false);
6667
jdoProperties.put(PropertyNames.PROPERTY_SCHEMA_VALIDATE_CONSTRAINTS, false);
6768

69+
// Required for non "select" SQL statements.
70+
jdoProperties.put(PropertyNames.PROPERTY_QUERY_SQL_ALLOWALL, true);
71+
6872
pmf = JDOHelper.getPersistenceManagerFactory(jdoProperties, classLoader);
6973
this.classLoader = classLoader;
7074
}
@@ -104,6 +108,17 @@ public void persistEntity(Object entity) {
104108
}
105109
}
106110

111+
public Object runQuery(String sql, Class<?> clazz, boolean unique) {
112+
PersistenceManager pm = pmf.getPersistenceManager();
113+
114+
Query<?> query = pm.newQuery("javax.jdo.query.SQL", sql);
115+
if (clazz != null) {
116+
query.setResultClass(clazz);
117+
}
118+
query.setUnique(unique);
119+
return query.execute();
120+
}
121+
107122
public <T> List<T> getAll(Class<T> clazz) {
108123
if (clazz == null) {
109124
throw new IllegalArgumentException("Class cannot be null.");

‎addOns/oast/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
99
### Added
1010
- API support.
1111
- Raise alerts for OAST interactions that happened in other sessions.
12+
- Options to trim the OAST permanent database.
1213

1314
### Changed
1415
- Depend on newer version of Database add-on.

‎addOns/oast/src/main/java/org/zaproxy/addon/oast/ExtensionOast.java‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ public class ExtensionOast extends ExtensionAdaptor {
7575
private final Map<String, OastService> services = new HashMap<>();
7676

7777
private OastOptionsPanel oastOptionsPanel;
78+
private BoastOptionsPanelTab boastOptionsPanelTab;
7879
private OastPanel oastPanel;
7980
private OastParam oastParam;
8081
private BoastService boastService;
@@ -129,8 +130,8 @@ public void hook(ExtensionHook extensionHook) {
129130

130131
if (hasView()) {
131132
extensionHook.getHookView().addOptionPanel(getOastOptionsPanel());
132-
getOastOptionsPanel().addServicePanel(new GeneralOastOptionsPanelTab());
133-
getOastOptionsPanel().addServicePanel(new BoastOptionsPanelTab(boastService));
133+
getOastOptionsPanel().addServicePanel(new GeneralOastOptionsPanelTab(this));
134+
getOastOptionsPanel().addServicePanel(getBoastOptionsPanelTab());
134135
getOastOptionsPanel().addServicePanel(new CallbackOptionsPanelTab(callbackService));
135136
getOastOptionsPanel().addServicePanel(new InteractshOptionsPanelTab(interactshService));
136137
extensionHook.getHookMenu().addPopupMenuItem(new OastInsertPayloadMenu(this));
@@ -154,13 +155,26 @@ public void optionsLoaded() {
154155
@Override
155156
public void postInit() {
156157
if (oastParam.isUsePermanentDatabase()) {
157-
getPermanentDatabase();
158+
trimDatabase(oastParam.getDaysToKeepRecords());
158159
}
160+
159161
boastService.startService();
160162
callbackService.startService();
161163
interactshService.startService();
162164
}
163165

166+
public void trimDatabase(int days) {
167+
getPermanentDatabase().trim(days);
168+
}
169+
170+
public void clearAllRecords() {
171+
getPermanentDatabase().clearAllRecords();
172+
boastService.clearRegisteredServers();
173+
if (hasView()) {
174+
ThreadUtils.invokeAndWaitHandled(() -> getBoastOptionsPanelTab().resetBoastServers());
175+
}
176+
}
177+
164178
private void optionsChanged(OptionsParam optionsParam) {
165179
getOastServices().values().forEach(OastService::fireOastStateChanged);
166180
if (!wasUsePermanentDatabase && oastParam.isUsePermanentDatabase()) {
@@ -240,6 +254,13 @@ private OastOptionsPanel getOastOptionsPanel() {
240254
return oastOptionsPanel;
241255
}
242256

257+
private BoastOptionsPanelTab getBoastOptionsPanelTab() {
258+
if (boastOptionsPanelTab == null) {
259+
boastOptionsPanelTab = new BoastOptionsPanelTab(boastService);
260+
}
261+
return boastOptionsPanelTab;
262+
}
263+
243264
public OastPanel getOastPanel() {
244265
if (oastPanel == null) {
245266
oastPanel = new OastPanel(this);

‎addOns/oast/src/main/java/org/zaproxy/addon/oast/OastApi.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,14 @@ public class OastApi extends ApiImplementor {
3535
private static final String PREFIX = "oast";
3636

3737
private static final String ACTION_SET_ACTIVE_SCAN_SERVICE = "setActiveScanService";
38+
private static final String ACTION_SET_DAYS_TO_KEEP_RECORDS = "setDaysToKeepRecords";
3839
private static final String ACTION_SET_BOAST_OPTIONS = "setBoastOptions";
3940
private static final String ACTION_SET_CALLBACK_OPTIONS = "setCallbackOptions";
4041
private static final String ACTION_SET_INTERACTSH_OPTIONS = "setInteractshOptions";
42+
4143
private static final String VIEW_GET_ACTIVE_SCAN_SERVICE = "getActiveScanService";
4244
private static final String VIEW_GET_SERVICES = "getServices";
45+
private static final String VIEW_GET_DAYS_TO_KEEP_RECORDS = "getDaysToKeepRecords";
4346
private static final String VIEW_GET_BOAST_OPTIONS = "getBoastOptions";
4447
private static final String VIEW_GET_CALLBACK_OPTIONS = "getCallbackOptions";
4548
private static final String VIEW_GET_INTERACTSH_OPTIONS = "getInteractshOptions";
@@ -51,6 +54,7 @@ public class OastApi extends ApiImplementor {
5154
private static final String PARAM_SERVER = "server";
5255
private static final String PARAM_POLL_IN_SECS = "pollInSecs";
5356
private static final String PARAM_PORT = "port";
57+
private static final String PARAM_DAYS = "days";
5458

5559
private ExtensionOast ext;
5660

@@ -68,6 +72,8 @@ public OastApi(ExtensionOast ext) {
6872
this.addApiView(new ApiView(VIEW_GET_BOAST_OPTIONS));
6973
this.addApiView(new ApiView(VIEW_GET_CALLBACK_OPTIONS));
7074
this.addApiView(new ApiView(VIEW_GET_INTERACTSH_OPTIONS));
75+
this.addApiView(new ApiView(VIEW_GET_DAYS_TO_KEEP_RECORDS));
76+
7177
this.addApiAction(new ApiAction(ACTION_SET_ACTIVE_SCAN_SERVICE, new String[] {PARAM_NAME}));
7278
this.addApiAction(
7379
new ApiAction(
@@ -80,6 +86,8 @@ public OastApi(ExtensionOast ext) {
8086
new ApiAction(
8187
ACTION_SET_INTERACTSH_OPTIONS,
8288
new String[] {PARAM_SERVER, PARAM_POLL_IN_SECS, PARAM_AUTH_TOKEN}));
89+
this.addApiAction(
90+
new ApiAction(ACTION_SET_DAYS_TO_KEEP_RECORDS, new String[] {PARAM_DAYS}));
8391
}
8492

8593
@Override
@@ -98,6 +106,12 @@ public ApiResponse handleApiAction(String name, JSONObject params) throws ApiExc
98106
ApiException.Type.ILLEGAL_PARAMETER, params.getString(PARAM_NAME));
99107
}
100108
break;
109+
case ACTION_SET_DAYS_TO_KEEP_RECORDS:
110+
int days = ApiUtils.getIntParam(params, PARAM_DAYS);
111+
ext.getParams().setDaysToKeepRecords(days);
112+
ext.trimDatabase(days);
113+
break;
114+
101115
case ACTION_SET_BOAST_OPTIONS:
102116
ext.getBoastService().getParam().setBoastUri(params.getString(PARAM_SERVER));
103117
ext.getBoastService()
@@ -140,6 +154,9 @@ public ApiResponse handleApiView(String name, JSONObject params) throws ApiExcep
140154
case VIEW_GET_ACTIVE_SCAN_SERVICE:
141155
OastService service = ext.getActiveScanOastService();
142156
return new ApiResponseElement(name, service != null ? service.getName() : "");
157+
case VIEW_GET_DAYS_TO_KEEP_RECORDS:
158+
return new ApiResponseElement(
159+
name, Integer.toString(ext.getParams().getDaysToKeepRecords()));
143160
case VIEW_GET_SERVICES:
144161
ApiResponseList servList = new ApiResponseList(name);
145162

‎addOns/oast/src/main/java/org/zaproxy/addon/oast/OastParam.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public class OastParam extends VersionedAbstractParam {
3030
PARAM_BASE_KEY + ".activeScanService";
3131
private static final String PARAM_USE_PERMANENT_DATABASE =
3232
PARAM_BASE_KEY + ".usePermanentDatabase";
33+
private static final String PARAM_DAYS_TO_KEEP_RECORDS = PARAM_BASE_KEY + ".daysToKeepRecords";
3334

3435
public static final String NO_ACTIVE_SCAN_SERVICE_SELECTED_OPTION = "None";
3536

@@ -41,6 +42,7 @@ public class OastParam extends VersionedAbstractParam {
4142

4243
private String activeScanServiceName;
4344
private boolean usePermanentDatabase;
45+
private int daysToKeepRecords;
4446

4547
public OastParam() {}
4648

@@ -62,11 +64,21 @@ public void setUsePermanentDatabase(boolean usePermanentDatabase) {
6264
getConfig().setProperty(PARAM_USE_PERMANENT_DATABASE, usePermanentDatabase);
6365
}
6466

67+
public int getDaysToKeepRecords() {
68+
return daysToKeepRecords;
69+
}
70+
71+
public void setDaysToKeepRecords(int daysToKeepRecords) {
72+
this.daysToKeepRecords = daysToKeepRecords;
73+
getConfig().setProperty(PARAM_DAYS_TO_KEEP_RECORDS, daysToKeepRecords);
74+
}
75+
6576
@Override
6677
protected void parseImpl() {
6778
activeScanServiceName =
6879
getString(PARAM_ACTIVE_SCAN_SERVICE_NAME, NO_ACTIVE_SCAN_SERVICE_SELECTED_OPTION);
6980
usePermanentDatabase = getBoolean(PARAM_USE_PERMANENT_DATABASE, true);
81+
daysToKeepRecords = getInteger(PARAM_DAYS_TO_KEEP_RECORDS, 45);
7082
}
7183

7284
@Override

‎addOns/oast/src/main/java/org/zaproxy/addon/oast/internal/OastPermanentDatabase.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,4 +68,64 @@ public AlertEntity getAlertForPayload(String payload) {
6868
}
6969
return null;
7070
}
71+
72+
public void trim(int days) {
73+
if (days <= 0) {
74+
return;
75+
}
76+
LOGGER.debug("Trimming records older than {} days", days);
77+
try {
78+
// Yes, passing parameters in this way is horrible!
79+
// But I couldn't get the parameters working the "correct" way and we know its
80+
// definitely a positive int..
81+
String dateClause = "DATE_SUB(CURRENT_TIMESTAMP, INTERVAL " + days + " DAY)";
82+
83+
Object res =
84+
runQuery(
85+
"DELETE FROM BOAST WHERE REGISTERED_TIMESTAMP < " + dateClause,
86+
null,
87+
false);
88+
if (Integer.parseInt(res.toString()) > 0) {
89+
LOGGER.info("Number of old BOAST records trimmed: {}", res);
90+
}
91+
92+
res = runQuery("DELETE FROM ALERT WHERE CREATETIMESTAMP < " + dateClause, null, false);
93+
if (Integer.parseInt(res.toString()) > 0) {
94+
LOGGER.info("Number of old ALERT records trimmed: {}", res);
95+
}
96+
97+
res =
98+
runQuery(
99+
"DELETE FROM MESSAGE WHERE CREATETIMESTAMP < " + dateClause,
100+
null,
101+
false);
102+
if (Integer.parseInt(res.toString()) > 0) {
103+
LOGGER.info("Number of old MESSAGE records trimmed: {}", res);
104+
}
105+
106+
} catch (Exception e) {
107+
LOGGER.error("Failed to trim OAST permanent db", e);
108+
}
109+
}
110+
111+
public void clearAllRecords() {
112+
try {
113+
Object res = runQuery("DELETE FROM BOAST", null, false);
114+
if (Integer.parseInt(res.toString()) > 0) {
115+
LOGGER.info("Number of old BOAST records trimmed: {}", res);
116+
}
117+
118+
res = runQuery("DELETE FROM ALERT", null, false);
119+
if (Integer.parseInt(res.toString()) > 0) {
120+
LOGGER.info("Number of old ALERT records trimmed: {}", res);
121+
}
122+
123+
res = runQuery("DELETE FROM MESSAGE", null, false);
124+
if (Integer.parseInt(res.toString()) > 0) {
125+
LOGGER.info("Number of old MESSAGE records trimmed: {}", res);
126+
}
127+
} catch (Exception e) {
128+
LOGGER.error("Failed to trim db", e);
129+
}
130+
}
71131
}

‎addOns/oast/src/main/java/org/zaproxy/addon/oast/services/boast/BoastOptionsPanelTab.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,13 +140,17 @@ private void registerButtonAction() {
140140
}
141141
}
142142

143+
public void resetBoastServers() {
144+
boastServersTableModel = null;
145+
getBoastServersTable().setModel(getBoastServersTableModel());
146+
}
147+
143148
@Override
144149
public void initParam(OptionsParam options) {
145150
final BoastParam param = options.getParamSet(BoastParam.class);
146151
getBoastUri().setText(param.getBoastUri());
147152
getPollingFrequencySpinner().setValue(param.getPollingFrequency());
148-
boastServersTableModel = null;
149-
getBoastServersTable().setModel(getBoastServersTableModel());
153+
resetBoastServers();
150154
}
151155

152156
@Override

‎addOns/oast/src/main/java/org/zaproxy/addon/oast/services/boast/BoastService.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,10 @@ public List<BoastServer> getRegisteredServers() {
111111
return registeredServers;
112112
}
113113

114+
public void clearRegisteredServers() {
115+
registeredServers.clear();
116+
}
117+
114118
public BoastServer register(String uri) throws IOException {
115119
getParam().setBoastUri(uri);
116120
return register();

0 commit comments

Comments
 (0)