Skip to content

Enhance pagination and search functionality for nodes in JSON endpoints #734

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
javuto merged 5 commits into from
Oct 14, 2025
Merged

Enhance pagination and search functionality for nodes in JSON endpoints #734

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
62e3a7d
Implement deferred Select2 initialization for tag modal
zhuoyuan-liu Oct 14, 2025
4b88b81
Merge remote-tracking branch 'upstream/main' into fix-ui-db
zhuoyuan-liu Oct 14, 2025
094041d
Enhance pagination and search functionality for nodes in JSON endpoints
zhuoyuan-liu Oct 14, 2025
1b87121
Remove fallback as Select2 is already globally included
zhuoyuan-liu Oct 14, 2025
c75f392
Merge branch 'fix-ui-db' into pagination
zhuoyuan-liu Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 54 additions & 18 deletions cmd/admin/handlers/json-nodes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,37 +167,44 @@ func (h *HandlersAdmin) JSONEnvironmentPagingHandler(w http.ResponseWriter, r *h
start, _ := strconv.Atoi(r.URL.Query().Get("start"))
length, _ := strconv.Atoi(r.URL.Query().Get("length"))
searchValue := r.URL.Query().Get("search")
// Get total nodes count
totalCount, err := h.Nodes.CountAllByEnv(env.ID)

// DB-level counts
totalCount, err := h.Nodes.CountByEnvTarget(env.Name, target, h.Settings.InactiveHours(settings.NoEnvironmentID))
if err != nil {
log.Err(err).Msg("error getting total nodes count")
log.Err(err).Msg("error counting total nodes")
return
}
var nodes []nodes.OsqueryNode
// If there is a search value, run the search
var filteredCount int64
var nodesSlice []nodes.OsqueryNode
hours := h.Settings.InactiveHours(settings.NoEnvironmentID)
// Ordering (DataTables sends order[0][column], order[0][dir])
orderColIdxStr := r.URL.Query().Get("order[0][column]")
orderDir := r.URL.Query().Get("order[0][dir]")
colName := mapDTColumnToDB(orderColIdxStr)
desc := (orderDir == "desc")
if searchValue != "" {
nodes, err = h.Nodes.SearchByEnv(env.Name, searchValue, target, h.Settings.InactiveHours(settings.NoEnvironmentID))
// Count filtered first
filteredCount, err = h.Nodes.CountSearchByEnv(env.Name, searchValue, target, hours)
if err != nil {
log.Err(err).Msg("error searching nodes")
log.Err(err).Msg("error counting search nodes")
return
}
nodesSlice, err = h.Nodes.SearchByEnvPage(env.Name, searchValue, target, hours, start, length, colName, desc)
if err != nil {
log.Err(err).Msg("error searching nodes page")
return
}
} else {
nodes, err = h.Nodes.GetByEnvLimit(env.Name, target, h.Settings.InactiveHours(settings.NoEnvironmentID))
filteredCount = totalCount
nodesSlice, err = h.Nodes.GetByEnvPage(env.Name, target, hours, start, length, colName, desc)
if err != nil {
log.Err(err).Msg("error getting nodes")
log.Err(err).Msg("error getting nodes page")
return
}
}
// Pagination, it can be done more efficiently in the DB, but this is ok for now
end := min(start+length, len(nodes))
if start > end {
start = end
}
filteredCount := len(nodes)
nodes = nodes[start:end]
// Prepare data to be returned
nJSON := []NodeJSON{}
for _, n := range nodes {
for _, n := range nodesSlice {
nj := NodeJSON{
UUID: n.UUID,
Username: n.Username,
Expand All @@ -220,9 +227,38 @@ func (h *HandlersAdmin) JSONEnvironmentPagingHandler(w http.ResponseWriter, r *h
returned := PaginatedNodes{
Draw: draw,
Total: int(totalCount),
Filtered: filteredCount,
Filtered: int(filteredCount),
Data: nJSON,
}
// Serve JSON
utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, returned)
}

// mapDTColumnToDB maps DataTables column index (as string) to actual DB column name.
// DataTables columns order in the UI:
// 0 checkbox,1 uuid,2 username(last user),3 localname,4 ip,5 platform,6 version(platform_version),7 osquery,8 lastseen,9 firstseen
// We only allow ordering on a safe subset of real DB columns.
func mapDTColumnToDB(idx string) string {
switch idx {
case "1":
return "uuid"
case "2":
return "username"
case "3":
return "localname"
case "4":
return "ip_address"
case "5":
return "platform"
case "6":
return "platform_version"
case "7":
return "osquery_version"
case "8":
return "last_seen"
case "9":
return "created_at" // first seen
default:
return "" // fallback to default ordering in caller
}
}
23 changes: 16 additions & 7 deletions cmd/admin/templates/table.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,17 @@ <h4 class="modal-title">Tag nodes</h4>
{{ if .OptimizeUI }}
url: "/paginated-json/{{ .Selector }}/{{ .SelectorName }}/{{ .Target }}",
data: function(d) {
// Explicitly forward ordering parameters expected by server handler
// while preserving the simplified search parameter name already used server-side.
var orderCol = (d.order && d.order.length) ? d.order[0].column : 8; // default Last Seen
var orderDir = (d.order && d.order.length) ? d.order[0].dir : 'desc';
return {
draw: d.draw,
start: d.start,
length: d.length,
search: d.search.value
search: d.search.value,
'order[0][column]': orderCol,
'order[0][dir]': orderDir
};
},
{{ else }}
Expand Down Expand Up @@ -411,12 +417,15 @@ <h4 class="modal-title">Tag nodes</h4>
$(this).find('#carve').focus();
});

// Select2 initialization
var tagsSelect = $('#modal_tags').select2({
theme: "classic"
});
tagsSelect.on("select2:select", function(e){
$('#add_tags').append(new Option(e.params.data.text, e.params.data.text));
// Deferred Select2 initialization
var _tagsSelectReady = false;
$('#tagModal').one('shown.bs.modal', function(){
if(_tagsSelectReady) return;
var tagsSelect = $('#modal_tags').select2({ theme: 'classic' });
tagsSelect.on('select2:select', function(e){
$('#add_tags').append(new Option(e.params.data.text, e.params.data.text));
});
_tagsSelectReady = true;
});
});
</script>
Expand Down
78 changes: 72 additions & 6 deletions pkg/nodes/nodes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,20 +198,46 @@ func (n *NodeManager) GetByEnv(env, target string, hours int64) ([]OsqueryNode,
return nodes, nil
}

// GetByEnvLimit to retrieve target nodes by environment with limit and offset
func (n *NodeManager) GetByEnvLimit(env, target string, hours int64) ([]OsqueryNode, error) {
// GetByEnvPage retrieves a page of nodes by environment applying target filters using LIMIT/OFFSET
func (n *NodeManager) GetByEnvPage(env, target string, hours int64, offset, limit int, orderBy string, desc bool) ([]OsqueryNode, error) {
var nodes []OsqueryNode
// Build query with base condition
if limit <= 0 { // safety default
limit = 25
}
if limit > 500 { // cap to avoid abuse
limit = 500
}
if offset < 0 {
offset = 0
}
Comment on lines +204 to +212
Copy link

Copilot AI Oct 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The limit validation logic is duplicated between GetByEnvPage and SearchByEnvPage methods. Consider extracting this into a helper function to avoid code duplication.

Copilot uses AI. Check for mistakes.
query := n.DB.Where("environment = ?", env)
// Apply active/inactive filtering
query = ApplyNodeTarget(query, target, hours)
// Execute query
if err := query.Find(&nodes).Error; err != nil {
// Default ordering only if client did not request a specific column
orderExpr := "last_seen DESC"
if orderBy != "" {
direction := "ASC"
if desc {
direction = "DESC"
}
orderExpr = orderBy + " " + direction
}
if err := query.Order(orderExpr).Offset(offset).Limit(limit).Find(&nodes).Error; err != nil {
return nodes, err
}
return nodes, nil
}

// CountByEnvTarget counts nodes for an environment after applying target (active/inactive/all)
func (n *NodeManager) CountByEnvTarget(env string, target string, hours int64) (int64, error) {
var count int64
query := n.DB.Model(&OsqueryNode{}).Where("environment = ?", env)
query = ApplyNodeTarget(query, target, hours)
if err := query.Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}

// SearchByEnv to search nodes by environment and search term
func (n *NodeManager) SearchByEnv(env, term, target string, hours int64) ([]OsqueryNode, error) {
var nodes []OsqueryNode
Expand All @@ -227,6 +253,46 @@ func (n *NodeManager) SearchByEnv(env, term, target string, hours int64) ([]Osqu
return nodes, nil
}

// SearchByEnvPage performs a paginated search
func (n *NodeManager) SearchByEnvPage(env, term, target string, hours int64, offset, limit int, orderBy string, desc bool) ([]OsqueryNode, error) {
if limit <= 0 {
limit = 25
} else if limit > 500 {
limit = 500
}
if offset < 0 {
offset = 0
}
var nodes []OsqueryNode
likeTerm := "%" + term + "%"
query := n.DB.Where("environment = ? AND (uuid LIKE ? OR hostname LIKE ? OR localname LIKE ? OR ip_address LIKE ? OR username LIKE ? OR osquery_user LIKE ? OR platform LIKE ? OR osquery_version LIKE ?)", env, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm)
Copy link

Copilot AI Oct 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The complex search query with multiple LIKE conditions is duplicated between SearchByEnvPage and CountSearchByEnv. Consider extracting this query building logic into a helper method to ensure consistency and maintainability.

Copilot uses AI. Check for mistakes.
query = ApplyNodeTarget(query, target, hours)
orderExpr := "last_seen DESC"
if orderBy != "" {
direction := "ASC"
if desc {
direction = "DESC"
}
orderExpr = orderBy + " " + direction
}
if err := query.Order(orderExpr).Offset(offset).Limit(limit).Find(&nodes).Error; err != nil {
return nodes, err
}
return nodes, nil
}

// CountSearchByEnv counts matching nodes for a search term with target filters
func (n *NodeManager) CountSearchByEnv(env, term, target string, hours int64) (int64, error) {
likeTerm := "%" + term + "%"
query := n.DB.Model(&OsqueryNode{}).Where("environment = ? AND (uuid LIKE ? OR hostname LIKE ? OR localname LIKE ? OR ip_address LIKE ? OR username LIKE ? OR osquery_user LIKE ? OR platform LIKE ? OR osquery_version LIKE ?)", env, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm)
Copy link

Copilot AI Oct 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The complex search query with multiple LIKE conditions is duplicated between SearchByEnvPage and CountSearchByEnv. Consider extracting this query building logic into a helper method to ensure consistency and maintainability.

Copilot uses AI. Check for mistakes.
query = ApplyNodeTarget(query, target, hours)
var count int64
if err := query.Count(&count).Error; err != nil {
return 0, err
}
return count, nil
}

// GetByPlatform to retrieve target nodes by platform
func (n *NodeManager) GetByPlatform(envID uint, platform, target string, hours int64) ([]OsqueryNode, error) {
var nodes []OsqueryNode
Expand Down
Loading