Skip to content

Commit 8268a3d

Browse files
javutojavuto
authored
Merge pull request #730 from jmpsec/pagination-query-results
Download distributed query results if > 100
2 parents 866e88f + 0e50767 commit 8268a3d

File tree

3 files changed

+192
-14
lines changed

3 files changed

+192
-14
lines changed

‎cmd/admin/handlers/json-logs.go‎

Lines changed: 104 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ type LogJSON struct {
4141

4242
// ReturnedQueryLogs to return a JSON with query logs
4343
type ReturnedQueryLogs struct {
44-
Data []QueryLogJSON `json:"data"`
44+
Data []QueryLogJSON `json:"data"`
45+
Download string `json:"download"`
4546
}
4647

4748
// QueryTargetNode to return the target of a on-demand query
@@ -168,6 +169,93 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)
168169

169170
// JSONQueryLogsHandler for JSON query logs by query name
170171
func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Request) {
172+
if h.DebugHTTPConfig.Enabled {
173+
utils.DebugHTTPDump(h.DebugHTTP, r, h.DebugHTTPConfig.ShowBody)
174+
}
175+
// Extract environment
176+
envVar := r.PathValue("env")
177+
if envVar == "" {
178+
log.Info().Msg("error getting environment")
179+
return
180+
}
181+
// Check if environment is valid
182+
if !h.Envs.Exists(envVar) {
183+
log.Info().Msgf("error unknown environment (%s)", envVar)
184+
return
185+
}
186+
// Get environment
187+
env, err := h.Envs.Get(envVar)
188+
if err != nil {
189+
log.Err(err).Msgf("error getting environment %s", envVar)
190+
return
191+
}
192+
// Get context data
193+
ctx := r.Context().Value(sessions.ContextKey(sessions.CtxSession)).(sessions.ContextValue)
194+
// Check permissions
195+
if !h.Users.CheckPermissions(ctx[sessions.CtxUser], users.QueryLevel, env.UUID) {
196+
log.Info().Msgf("%s has insufficient permissions", ctx[sessions.CtxUser])
197+
return
198+
}
199+
// Extract query name
200+
// FIXME verify name
201+
name := r.PathValue("name")
202+
if name == "" {
203+
log.Info().Msg("error getting name")
204+
return
205+
}
206+
// Iterate through targets to get logs
207+
queryLogJSON := []QueryLogJSON{}
208+
var downloadUrl string
209+
// Get logs
210+
if h.DBLogger != nil {
211+
queryLogs, err := h.DBLogger.QueryLogs(name)
212+
if err != nil {
213+
log.Err(err).Msg("error getting logs")
214+
return
215+
}
216+
// TODO customize max number of logs to show
217+
if h.OptimizedUI && len(queryLogs) > 100 {
218+
downloadUrl = "/json-download/query/" + envVar + "/" + name
219+
} else {
220+
// Prepare data to be returned
221+
for _, q := range queryLogs {
222+
// Get target node
223+
node, err := h.Nodes.GetByUUID(q.UUID)
224+
if err != nil {
225+
node.UUID = q.UUID
226+
node.Localname = ""
227+
}
228+
_c := CreationTimes{
229+
Display: utils.PastFutureTimes(q.CreatedAt),
230+
Timestamp: strconv.Itoa(int(q.CreatedAt.Unix())),
231+
}
232+
qData, err := json.Marshal(q.Data)
233+
if err != nil {
234+
log.Err(err).Msg("error serializing logs")
235+
continue
236+
}
237+
_l := QueryLogJSON{
238+
Created: _c,
239+
Target: QueryTargetNode{
240+
UUID: node.UUID,
241+
Name: node.Localname,
242+
},
243+
Data: string(qData),
244+
}
245+
queryLogJSON = append(queryLogJSON, _l)
246+
}
247+
}
248+
}
249+
returned := ReturnedQueryLogs{
250+
Data: queryLogJSON,
251+
Download: downloadUrl,
252+
}
253+
// Serialize and serve JSON
254+
utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, returned)
255+
}
256+
257+
// JSONDownloadQueryLogsHandler for JSON query logs by query name
258+
func (h *HandlersAdmin) JSONDownloadQueryLogsHandler(w http.ResponseWriter, r *http.Request) {
171259
if h.DebugHTTPConfig.Enabled {
172260
utils.DebugHTTPDump(h.DebugHTTP, r, h.DebugHTTPConfig.ShowBody)
173261
}
@@ -239,9 +327,20 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ
239327
queryLogJSON = append(queryLogJSON, _l)
240328
}
241329
}
242-
returned := ReturnedQueryLogs{
243-
Data: queryLogJSON,
330+
// Prepare JSON data for download
331+
jsonData, err := json.MarshalIndent(queryLogJSON, "", " ")
332+
if err != nil {
333+
log.Err(err).Msg("error marshaling query logs")
334+
utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusInternalServerError, nil)
335+
return
336+
}
337+
// Set headers for file download
338+
filename := name + "_logs.json"
339+
desc := "Logs for " + name
340+
utils.HTTPDownload(w, desc, filename, int64(len(jsonData)))
341+
// Write the file content
342+
w.WriteHeader(http.StatusOK)
343+
if _, err := w.Write(jsonData); err != nil {
344+
log.Err(err).Msg("error writing response")
244345
}
245-
// Serialize and serve JSON
246-
utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, returned)
247346
}

‎cmd/admin/main.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,11 @@ func osctrlAdminService() {
394394
adminMux.Handle(
395395
"GET /json/query/{env}/{name}",
396396
handlerAuthCheck(http.HandlerFunc(handlersAdmin.JSONQueryLogsHandler), flagParams.ConfigValues.Auth))
397+
if flagParams.OptimizeUI {
398+
adminMux.Handle(
399+
"GET /json-download/query/{env}/{name}",
400+
handlerAuthCheck(http.HandlerFunc(handlersAdmin.JSONDownloadQueryLogsHandler), flagParams.ConfigValues.Auth))
401+
}
397402
// Admin: JSON data for sidebar stats
398403
adminMux.Handle(
399404
"GET /json/stats/{target}/{identifier}",

‎cmd/admin/templates/queries-logs.html‎

Lines changed: 83 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@
1111
<main class="main">
1212
<div class="container-fluid">
1313
<div class="animated fadeIn">
14-
{{ if $leftmeta.Query }}
15-
{{ with .Query }}
14+
{{ if $leftmeta.Query }} {{ with .Query }}
1615
<div class="card mt-2">
1716
<div class="card-header">
1817
{{ if .Expired }}
@@ -68,6 +67,11 @@
6867
</table>
6968
<br />
7069
{{ if eq $serviceConfig.Logger "db" }}
70+
<div id="downloadLinkContainer" class="alert alert-info" style="display: none">
71+
<i class="fas fa-info-circle"></i>
72+
<strong>Large Result Set:</strong> The results are too large to display in the table.
73+
<a id="downloadLink" href="#" class="btn btn-sm btn-primary ml-2"> <i class="fas fa-download"></i> Download Full Results </a>
74+
</div>
7175
<table id="tableQueryLogs" class="table table-bordered table-striped" style="width: 100%">
7276
<input type="hidden" id="refresh_value" value="yes" />
7377
<thead>
@@ -83,15 +87,13 @@
8387
{{ end }}
8488
</div>
8589
</div>
86-
{{ end }}
87-
{{ else }}
90+
{{ end }} {{ else }}
8891
<div class="card mt-2">
8992
<div class="alert alert-danger" role="alert">
9093
<h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs not available</h4>
9194
</div>
9295
</div>
93-
{{ end }}
94-
96+
{{ end }}
9597
</div>
9698
</div>
9799
</main>
@@ -111,6 +113,7 @@ <h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs
111113
$(".card-header").addClass("bg-danger");
112114
};
113115
$.fn.dataTable.ext.ajax;
116+
var downloadUrl = ""; // Store download URL globally
114117
var tableQueryLogs = $("#tableQueryLogs").DataTable({
115118
initComplete: function (settings, json) {
116119
$(".card-header").removeClass("bg-danger");
@@ -122,6 +125,17 @@ <h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs
122125
url: "/json/query/{{ $leftmeta.EnvUUID }}/{{ .Name }}",
123126
dataSrc: function (json) {
124127
$("#status-card-header").removeClass("bg-danger");
128+
// Store download URL if available
129+
downloadUrl = json.download || "";
130+
131+
// Show/hide download link container
132+
if (downloadUrl && downloadUrl !== "") {
133+
$("#downloadLink").attr("href", downloadUrl);
134+
$("#downloadLinkContainer").show();
135+
} else {
136+
$("#downloadLinkContainer").hide();
137+
}
138+
125139
return json.data;
126140
},
127141
error: function (xhr, error, code) {
@@ -159,10 +173,70 @@ <h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs
159173
width: "75%",
160174
targets: 2,
161175
render: function (data, type, row, meta) {
162-
if (type === "display") {
163-
return "<pre>" + JSON.stringify(JSON.parse(JSON.parse(data)), null, 2) + "</pre>";
164-
} else {
176+
try {
177+
var rawData = JSON.parse(JSON.parse(data));
178+
var results = rawData.result;
179+
if (!Array.isArray(results) || results.length === 0) {
180+
return type === "display" ? '<div class="text-muted">No results</div>' : "";
181+
}
182+
// For filtering/searching, return all values as searchable text
183+
if (type === "filter" || type === "search") {
184+
var searchText = [];
185+
results.forEach(function (obj) {
186+
Object.keys(obj).forEach(function (key) {
187+
var value = obj[key];
188+
if (value !== undefined && value !== null) {
189+
if (typeof value === "object") {
190+
searchText.push(JSON.stringify(value));
191+
} else {
192+
searchText.push(String(value));
193+
}
194+
}
195+
});
196+
});
197+
return searchText.join(" ");
198+
}
199+
// For display, show the table
200+
if (type === "display") {
201+
// Get all unique keys from all objects
202+
var keys = [];
203+
results.forEach(function (obj) {
204+
Object.keys(obj).forEach(function (key) {
205+
if (keys.indexOf(key) === -1) {
206+
keys.push(key);
207+
}
208+
});
209+
});
210+
211+
// Build table
212+
var table = '<table class="table table-sm table-bordered table-hover" style="margin-bottom: 0;">';
213+
table += '<thead class="thead-light"><tr>';
214+
keys.forEach(function (key) {
215+
table += '<th style="font-size: 0.85rem;">' + key + "</th>";
216+
});
217+
table += "</tr></thead><tbody>";
218+
219+
results.forEach(function (obj) {
220+
table += "<tr>";
221+
keys.forEach(function (key) {
222+
var value = obj[key] !== undefined && obj[key] !== null ? obj[key] : "";
223+
// Handle nested objects/arrays
224+
if (typeof value === "object") {
225+
value = JSON.stringify(value);
226+
}
227+
table += '<td style="font-size: 0.85rem;">' + value + "</td>";
228+
});
229+
table += "</tr>";
230+
});
231+
232+
table += "</tbody></table>";
233+
return table;
234+
}
235+
236+
// For other types (sort, etc), return raw data
165237
return data;
238+
} catch (e) {
239+
return type === "display" ? '<div class="text-danger">Error parsing results: ' + e.message + "</div>" : "";
166240
}
167241
},
168242
},

0 commit comments

Comments
 (0)