Skip to content

Download distributed query results if > 100 #730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
javuto merged 1 commit into from
Oct 13, 2025
Merged

Download distributed query results if > 100 #730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 104 additions & 5 deletions cmd/admin/handlers/json-logs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ type LogJSON struct {

// ReturnedQueryLogs to return a JSON with query logs
type ReturnedQueryLogs struct {
Data []QueryLogJSON `json:"data"`
Data []QueryLogJSON `json:"data"`
Download string `json:"download"`
}

// QueryTargetNode to return the target of a on-demand query
Expand Down Expand Up @@ -168,6 +169,93 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)

// JSONQueryLogsHandler for JSON query logs by query name
func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Request) {
if h.DebugHTTPConfig.Enabled {
utils.DebugHTTPDump(h.DebugHTTP, r, h.DebugHTTPConfig.ShowBody)
}
// Extract environment
envVar := r.PathValue("env")
if envVar == "" {
log.Info().Msg("error getting environment")
return
}
// Check if environment is valid
if !h.Envs.Exists(envVar) {
log.Info().Msgf("error unknown environment (%s)", envVar)
return
}
// Get environment
env, err := h.Envs.Get(envVar)
if err != nil {
log.Err(err).Msgf("error getting environment %s", envVar)
return
}
// Get context data
ctx := r.Context().Value(sessions.ContextKey(sessions.CtxSession)).(sessions.ContextValue)
// Check permissions
if !h.Users.CheckPermissions(ctx[sessions.CtxUser], users.QueryLevel, env.UUID) {
log.Info().Msgf("%s has insufficient permissions", ctx[sessions.CtxUser])
return
}
// Extract query name
// FIXME verify name
name := r.PathValue("name")
if name == "" {
log.Info().Msg("error getting name")
return
}
// Iterate through targets to get logs
queryLogJSON := []QueryLogJSON{}
var downloadUrl string
// Get logs
if h.DBLogger != nil {
queryLogs, err := h.DBLogger.QueryLogs(name)
if err != nil {
log.Err(err).Msg("error getting logs")
return
}
// TODO customize max number of logs to show
if h.OptimizedUI && len(queryLogs) > 100 {
downloadUrl = "/json-download/query/" + envVar + "/" + name
} else {
// Prepare data to be returned
for _, q := range queryLogs {
// Get target node
node, err := h.Nodes.GetByUUID(q.UUID)
if err != nil {
node.UUID = q.UUID
node.Localname = ""
}
_c := CreationTimes{
Display: utils.PastFutureTimes(q.CreatedAt),
Timestamp: strconv.Itoa(int(q.CreatedAt.Unix())),
}
qData, err := json.Marshal(q.Data)
if err != nil {
log.Err(err).Msg("error serializing logs")
continue
}
_l := QueryLogJSON{
Created: _c,
Target: QueryTargetNode{
UUID: node.UUID,
Name: node.Localname,
},
Data: string(qData),
}
queryLogJSON = append(queryLogJSON, _l)
}
}
}
returned := ReturnedQueryLogs{
Data: queryLogJSON,
Download: downloadUrl,
}
// Serialize and serve JSON
utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, returned)
}

// JSONDownloadQueryLogsHandler for JSON query logs by query name
func (h *HandlersAdmin) JSONDownloadQueryLogsHandler(w http.ResponseWriter, r *http.Request) {
if h.DebugHTTPConfig.Enabled {
utils.DebugHTTPDump(h.DebugHTTP, r, h.DebugHTTPConfig.ShowBody)
}
Expand Down Expand Up @@ -239,9 +327,20 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ
queryLogJSON = append(queryLogJSON, _l)
}
}
returned := ReturnedQueryLogs{
Data: queryLogJSON,
// Prepare JSON data for download
jsonData, err := json.MarshalIndent(queryLogJSON, "", " ")
if err != nil {
log.Err(err).Msg("error marshaling query logs")
utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusInternalServerError, nil)
return
}
// Set headers for file download
filename := name + "_logs.json"
desc := "Logs for " + name
utils.HTTPDownload(w, desc, filename, int64(len(jsonData)))
// Write the file content
w.WriteHeader(http.StatusOK)
if _, err := w.Write(jsonData); err != nil {
log.Err(err).Msg("error writing response")
}
// Serialize and serve JSON
utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, returned)
}
5 changes: 5 additions & 0 deletions cmd/admin/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,11 @@ func osctrlAdminService() {
adminMux.Handle(
"GET /json/query/{env}/{name}",
handlerAuthCheck(http.HandlerFunc(handlersAdmin.JSONQueryLogsHandler), flagParams.ConfigValues.Auth))
if flagParams.OptimizeUI {
adminMux.Handle(
"GET /json-download/query/{env}/{name}",
handlerAuthCheck(http.HandlerFunc(handlersAdmin.JSONDownloadQueryLogsHandler), flagParams.ConfigValues.Auth))
}
// Admin: JSON data for sidebar stats
adminMux.Handle(
"GET /json/stats/{target}/{identifier}",
Expand Down
92 changes: 83 additions & 9 deletions cmd/admin/templates/queries-logs.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@
<main class="main">
<div class="container-fluid">
<div class="animated fadeIn">
{{ if $leftmeta.Query }}
{{ with .Query }}
{{ if $leftmeta.Query }} {{ with .Query }}
<div class="card mt-2">
<div class="card-header">
{{ if .Expired }}
Expand Down Expand Up @@ -68,6 +67,11 @@
</table>
<br />
{{ if eq $serviceConfig.Logger "db" }}
<div id="downloadLinkContainer" class="alert alert-info" style="display: none">
<i class="fas fa-info-circle"></i>
<strong>Large Result Set:</strong> The results are too large to display in the table.
<a id="downloadLink" href="#" class="btn btn-sm btn-primary ml-2"> <i class="fas fa-download"></i> Download Full Results </a>
</div>
<table id="tableQueryLogs" class="table table-bordered table-striped" style="width: 100%">
<input type="hidden" id="refresh_value" value="yes" />
<thead>
Expand All @@ -83,15 +87,13 @@
{{ end }}
</div>
</div>
{{ end }}
{{ else }}
{{ end }} {{ else }}
<div class="card mt-2">
<div class="alert alert-danger" role="alert">
<h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs not available</h4>
</div>
</div>
{{ end }}

{{ end }}
</div>
</div>
</main>
Expand All @@ -111,6 +113,7 @@ <h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs
$(".card-header").addClass("bg-danger");
};
$.fn.dataTable.ext.ajax;
var downloadUrl = ""; // Store download URL globally
var tableQueryLogs = $("#tableQueryLogs").DataTable({
initComplete: function (settings, json) {
$(".card-header").removeClass("bg-danger");
Expand All @@ -122,6 +125,17 @@ <h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs
url: "/json/query/{{ $leftmeta.EnvUUID }}/{{ .Name }}",
dataSrc: function (json) {
$("#status-card-header").removeClass("bg-danger");
// Store download URL if available
downloadUrl = json.download || "";

// Show/hide download link container
if (downloadUrl && downloadUrl !== "") {
$("#downloadLink").attr("href", downloadUrl);
$("#downloadLinkContainer").show();
} else {
$("#downloadLinkContainer").hide();
}

return json.data;
},
error: function (xhr, error, code) {
Expand Down Expand Up @@ -159,10 +173,70 @@ <h4 class="alert-heading"><i class="fas fa-exclamation-triangle"></i> Query logs
width: "75%",
targets: 2,
render: function (data, type, row, meta) {
if (type === "display") {
return "<pre>" + JSON.stringify(JSON.parse(JSON.parse(data)), null, 2) + "</pre>";
} else {
try {
var rawData = JSON.parse(JSON.parse(data));
var results = rawData.result;
if (!Array.isArray(results) || results.length === 0) {
return type === "display" ? '<div class="text-muted">No results</div>' : "";
}
// For filtering/searching, return all values as searchable text
if (type === "filter" || type === "search") {
var searchText = [];
results.forEach(function (obj) {
Object.keys(obj).forEach(function (key) {
var value = obj[key];
if (value !== undefined && value !== null) {
if (typeof value === "object") {
searchText.push(JSON.stringify(value));
} else {
searchText.push(String(value));
}
}
});
});
return searchText.join(" ");
}
// For display, show the table
if (type === "display") {
// Get all unique keys from all objects
var keys = [];
results.forEach(function (obj) {
Object.keys(obj).forEach(function (key) {
if (keys.indexOf(key) === -1) {
keys.push(key);
}
});
});

// Build table
var table = '<table class="table table-sm table-bordered table-hover" style="margin-bottom: 0;">';
table += '<thead class="thead-light"><tr>';
keys.forEach(function (key) {
table += '<th style="font-size: 0.85rem;">' + key + "</th>";
});
table += "</tr></thead><tbody>";

results.forEach(function (obj) {
table += "<tr>";
keys.forEach(function (key) {
var value = obj[key] !== undefined && obj[key] !== null ? obj[key] : "";
// Handle nested objects/arrays
if (typeof value === "object") {
value = JSON.stringify(value);
}
table += '<td style="font-size: 0.85rem;">' + value + "</td>";
});
table += "</tr>";
});

table += "</tbody></table>";
return table;
}

// For other types (sort, etc), return raw data
return data;
} catch (e) {
return type === "display" ? '<div class="text-danger">Error parsing results: ' + e.message + "</div>" : "";
}
},
},
Expand Down
Loading