Merge pull request #734 from zhuoyuan-liu/pagination

Enhance pagination and search functionality for nodes in JSON endpoints

Author: javuto

cmd/admin/handlers/json-nodes.go

@@ -167,37 +167,44 @@ func (h *HandlersAdmin) JSONEnvironmentPagingHandler(w http.ResponseWriter, r *h
 	start, _ := strconv.Atoi(r.URL.Query().Get("start"))
 	length, _ := strconv.Atoi(r.URL.Query().Get("length"))
 	searchValue := r.URL.Query().Get("search")
-	// Get total nodes count
-	totalCount, err := h.Nodes.CountAllByEnv(env.ID)
+
+	// DB-level counts
+	totalCount, err := h.Nodes.CountByEnvTarget(env.Name, target, h.Settings.InactiveHours(settings.NoEnvironmentID))
 	if err != nil {
-		log.Err(err).Msg("error getting total nodes count")
+		log.Err(err).Msg("error counting total nodes")
 		return
 	}
-	var nodes []nodes.OsqueryNode
-	// If there is a search value, run the search
+	var filteredCount int64
+	var nodesSlice []nodes.OsqueryNode
+	hours := h.Settings.InactiveHours(settings.NoEnvironmentID)
+	// Ordering (DataTables sends order[0][column], order[0][dir])
+	orderColIdxStr := r.URL.Query().Get("order[0][column]")
+	orderDir := r.URL.Query().Get("order[0][dir]")
+	colName := mapDTColumnToDB(orderColIdxStr)
+	desc := (orderDir == "desc")
 	if searchValue != "" {
-		nodes, err = h.Nodes.SearchByEnv(env.Name, searchValue, target, h.Settings.InactiveHours(settings.NoEnvironmentID))
+		// Count filtered first
+		filteredCount, err = h.Nodes.CountSearchByEnv(env.Name, searchValue, target, hours)
 		if err != nil {
-			log.Err(err).Msg("error searching nodes")
+			log.Err(err).Msg("error counting search nodes")
+			return
+		}
+		nodesSlice, err = h.Nodes.SearchByEnvPage(env.Name, searchValue, target, hours, start, length, colName, desc)
+		if err != nil {
+			log.Err(err).Msg("error searching nodes page")
 			return
 		}
 	} else {
-		nodes, err = h.Nodes.GetByEnvLimit(env.Name, target, h.Settings.InactiveHours(settings.NoEnvironmentID))
+		filteredCount = totalCount
+		nodesSlice, err = h.Nodes.GetByEnvPage(env.Name, target, hours, start, length, colName, desc)
 		if err != nil {
-			log.Err(err).Msg("error getting nodes")
+			log.Err(err).Msg("error getting nodes page")
 			return
 		}
 	}
-	// Pagination, it can be done more efficiently in the DB, but this is ok for now
-	end := min(start+length, len(nodes))
-	if start > end {
-		start = end
-	}
-	filteredCount := len(nodes)
-	nodes = nodes[start:end]
 	// Prepare data to be returned
 	nJSON := []NodeJSON{}
-	for _, n := range nodes {
+	for _, n := range nodesSlice {
 		nj := NodeJSON{
 			UUID:      n.UUID,
 			Username:  n.Username,
@@ -220,9 +227,38 @@ func (h *HandlersAdmin) JSONEnvironmentPagingHandler(w http.ResponseWriter, r *h
 	returned := PaginatedNodes{
 		Draw:     draw,
 		Total:    int(totalCount),
-		Filtered: filteredCount,
+		Filtered: int(filteredCount),
 		Data:     nJSON,
 	}
 	// Serve JSON
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, returned)
 }
+
+// mapDTColumnToDB maps DataTables column index (as string) to actual DB column name.
+// DataTables columns order in the UI:
+// 0 checkbox,1 uuid,2 username(last user),3 localname,4 ip,5 platform,6 version(platform_version),7 osquery,8 lastseen,9 firstseen
+// We only allow ordering on a safe subset of real DB columns.
+func mapDTColumnToDB(idx string) string {
+	switch idx {
+	case "1":
+		return "uuid"
+	case "2":
+		return "username"
+	case "3":
+		return "localname"
+	case "4":
+		return "ip_address"
+	case "5":
+		return "platform"
+	case "6":
+		return "platform_version"
+	case "7":
+		return "osquery_version"
+	case "8":
+		return "last_seen"
+	case "9":
+		return "created_at" // first seen
+	default:
+		return "" // fallback to default ordering in caller
+	}
+}

cmd/admin/templates/table.html

@@ -127,11 +127,17 @@ <h4 class="modal-title">Tag nodes</h4>
           {{ if .OptimizeUI }}
             url: "/paginated-json/{{ .Selector }}/{{ .SelectorName }}/{{ .Target }}",
             data: function(d) {
+              // Explicitly forward ordering parameters expected by server handler
+              // while preserving the simplified search parameter name already used server-side.
+              var orderCol = (d.order && d.order.length) ? d.order[0].column : 8; // default Last Seen
+              var orderDir = (d.order && d.order.length) ? d.order[0].dir : 'desc';
               return {
                 draw: d.draw,
                 start: d.start,
                 length: d.length,
-                search: d.search.value
+                search: d.search.value,
+                'order[0][column]': orderCol,
+                'order[0][dir]': orderDir
               };
             },
           {{ else }}

pkg/nodes/nodes.go

@@ -198,20 +198,46 @@ func (n *NodeManager) GetByEnv(env, target string, hours int64) ([]OsqueryNode,
 	return nodes, nil
 }
 
-// GetByEnvLimit to retrieve target nodes by environment with limit and offset
-func (n *NodeManager) GetByEnvLimit(env, target string, hours int64) ([]OsqueryNode, error) {
+// GetByEnvPage retrieves a page of nodes by environment applying target filters using LIMIT/OFFSET
+func (n *NodeManager) GetByEnvPage(env, target string, hours int64, offset, limit int, orderBy string, desc bool) ([]OsqueryNode, error) {
 	var nodes []OsqueryNode
-	// Build query with base condition
+	if limit <= 0 { // safety default
+		limit = 25
+	}
+	if limit > 500 { // cap to avoid abuse
+		limit = 500
+	}
+	if offset < 0 {
+		offset = 0
+	}
 	query := n.DB.Where("environment = ?", env)
-	// Apply active/inactive filtering
 	query = ApplyNodeTarget(query, target, hours)
-	// Execute query
-	if err := query.Find(&nodes).Error; err != nil {
+	// Default ordering only if client did not request a specific column
+	orderExpr := "last_seen DESC"
+	if orderBy != "" {
+		direction := "ASC"
+		if desc {
+			direction = "DESC"
+		}
+		orderExpr = orderBy + " " + direction
+	}
+	if err := query.Order(orderExpr).Offset(offset).Limit(limit).Find(&nodes).Error; err != nil {
 		return nodes, err
 	}
 	return nodes, nil
 }
 
+// CountByEnvTarget counts nodes for an environment after applying target (active/inactive/all)
+func (n *NodeManager) CountByEnvTarget(env string, target string, hours int64) (int64, error) {
+	var count int64
+	query := n.DB.Model(&OsqueryNode{}).Where("environment = ?", env)
+	query = ApplyNodeTarget(query, target, hours)
+	if err := query.Count(&count).Error; err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
 // SearchByEnv to search nodes by environment and search term
 func (n *NodeManager) SearchByEnv(env, term, target string, hours int64) ([]OsqueryNode, error) {
 	var nodes []OsqueryNode
@@ -227,6 +253,46 @@ func (n *NodeManager) SearchByEnv(env, term, target string, hours int64) ([]Osqu
 	return nodes, nil
 }
 
+// SearchByEnvPage performs a paginated search
+func (n *NodeManager) SearchByEnvPage(env, term, target string, hours int64, offset, limit int, orderBy string, desc bool) ([]OsqueryNode, error) {
+	if limit <= 0 {
+		limit = 25
+	} else if limit > 500 {
+		limit = 500
+	}
+	if offset < 0 {
+		offset = 0
+	}
+	var nodes []OsqueryNode
+	likeTerm := "%" + term + "%"
+	query := n.DB.Where("environment = ? AND (uuid LIKE ? OR hostname LIKE ? OR localname LIKE ? OR ip_address LIKE ? OR username LIKE ? OR osquery_user LIKE ? OR platform LIKE ? OR osquery_version LIKE ?)", env, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm)
+	query = ApplyNodeTarget(query, target, hours)
+	orderExpr := "last_seen DESC"
+	if orderBy != "" {
+		direction := "ASC"
+		if desc {
+			direction = "DESC"
+		}
+		orderExpr = orderBy + " " + direction
+	}
+	if err := query.Order(orderExpr).Offset(offset).Limit(limit).Find(&nodes).Error; err != nil {
+		return nodes, err
+	}
+	return nodes, nil
+}
+
+// CountSearchByEnv counts matching nodes for a search term with target filters
+func (n *NodeManager) CountSearchByEnv(env, term, target string, hours int64) (int64, error) {
+	likeTerm := "%" + term + "%"
+	query := n.DB.Model(&OsqueryNode{}).Where("environment = ? AND (uuid LIKE ? OR hostname LIKE ? OR localname LIKE ? OR ip_address LIKE ? OR username LIKE ? OR osquery_user LIKE ? OR platform LIKE ? OR osquery_version LIKE ?)", env, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm, likeTerm)
+	query = ApplyNodeTarget(query, target, hours)
+	var count int64
+	if err := query.Count(&count).Error; err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
 // GetByPlatform to retrieve target nodes by platform
 func (n *NodeManager) GetByPlatform(envID uint, platform, target string, hours int64) ([]OsqueryNode, error) {
 	var nodes []OsqueryNode