Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions lib/sdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -358,3 +358,26 @@ func wait(wg *sync.WaitGroup) <-chan struct{} {
}()
return ch
}

// GetClusterTemplateIDs returns the template IDs for a given cluster ID
// Returns nil if the cluster ID doesn't exist or engine hasn't executed yet
func (e *NucleiEngine) GetClusterTemplateIDs(clusterID string) []string {
if e.executerOpts == nil || e.executerOpts.ClusterMappings == nil {
return nil
}
templateIDs, ok := e.executerOpts.ClusterMappings.Get(clusterID)
if !ok {
return nil
}
return templateIDs
}

// GetAllClusterMappings returns all cluster mappings
// Returns nil if engine hasn't executed yet
func (e *NucleiEngine) GetAllClusterMappings() map[string][]string {
if e.executerOpts == nil || e.executerOpts.ClusterMappings == nil {
return nil
}

return e.executerOpts.ClusterMappings.GetAll()
}
7 changes: 6 additions & 1 deletion pkg/core/execute_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,12 @@ func (e *Engine) ExecuteScanWithOpts(ctx context.Context, templatesList []*templ
var finalTemplates []*templates.Template
clusterCount := 0
if !noCluster {
finalTemplates, clusterCount = templates.ClusterTemplates(templatesList, e.executerOpts)
var clusterMappings map[string][]string
finalTemplates, clusterCount, clusterMappings = templates.ClusterTemplates(templatesList, e.executerOpts)
// Store cluster mappings in executerOpts for SDK access (thread-safe)
if clusterMappings != nil {
e.executerOpts.ClusterMappings = types.NewClusterMappingsMap(clusterMappings)
}
} else {
finalTemplates = templatesList
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/protocols/common/automaticscan/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func LoadTemplatesWithTags(opts Options, templateDirs []string, tags []string, l
if !opts.ExecuterOpts.Options.DisableClustering {
// cluster and reduce requests
totalReqBeforeCluster := getRequestCount(finalTemplates) * int(opts.Target.Count())
finalTemplates, clusterCount := templates.ClusterTemplates(finalTemplates, opts.ExecuterOpts)
finalTemplates, clusterCount, _ := templates.ClusterTemplates(finalTemplates, opts.ExecuterOpts)
totalReqAfterClustering := getRequestCount(finalTemplates) * int(opts.Target.Count())
if totalReqAfterClustering < totalReqBeforeCluster && logInfo {
opts.ExecuterOpts.Logger.Info().Msgf("Automatic scan tech-detect: Templates clustered: %d (Reduced %d Requests)", clusterCount, totalReqBeforeCluster-totalReqAfterClustering)
Expand Down
4 changes: 4 additions & 0 deletions pkg/protocols/protocols.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,8 @@ type ExecutorOptions struct {
Logger *gologger.Logger
// CustomFastdialer is a fastdialer dialer instance
CustomFastdialer *fastdialer.Dialer
// ClusterMappings stores cluster ID to template IDs mapping during execution
ClusterMappings *templateTypes.ClusterMappingsMap
}

// todo: centralizing components is not feasible with current clogged architecture
Expand Down Expand Up @@ -310,6 +312,7 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
GlobalMatchers: e.GlobalMatchers,
Logger: e.Logger,
}
copy.ClusterMappings = e.ClusterMappings.Copy()
copy.CreateTemplateCtxStore()
return copy
}
Expand Down Expand Up @@ -482,4 +485,5 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
e.GlobalMatchers = n.GlobalMatchers
e.Logger = n.Logger
e.CustomFastdialer = n.CustomFastdialer
e.ClusterMappings = n.ClusterMappings.Copy()
}
18 changes: 15 additions & 3 deletions pkg/templates/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,17 @@ func ClusterID(templates []*Template) string {
return cryptoutil.SHA256Sum(ids)
}

func ClusterTemplates(templatesList []*Template, options *protocols.ExecutorOptions) ([]*Template, int) {
// ClusterTemplates clusters templates and returns:
// - the final list of templates (with clustered templates merged)
// - the count of templates that were clustered
// - a map of cluster ID to the original template IDs in each cluster
func ClusterTemplates(templatesList []*Template, options *protocols.ExecutorOptions) ([]*Template, int, map[string][]string) {
if options.Options.OfflineHTTP || options.Options.DisableClustering {
return templatesList, 0
return templatesList, 0, nil
}

var clusterCount int
clusterMappings := make(map[string][]string)

finalTemplatesList := make([]*Template, 0, len(templatesList))
clusters := Cluster(templatesList)
Expand All @@ -131,6 +136,13 @@ func ClusterTemplates(templatesList []*Template, options *protocols.ExecutorOpti
executerOpts := options
clusterID := fmt.Sprintf("cluster-%s", ClusterID(cluster))

// Collect all template IDs in this cluster
clusterTemplateIDs := make([]string, len(cluster))
for i, tpl := range cluster {
clusterTemplateIDs[i] = tpl.ID
}
clusterMappings[clusterID] = clusterTemplateIDs

for _, req := range cluster[0].RequestsDNS {
req.Options().TemplateID = clusterID
}
Expand All @@ -154,7 +166,7 @@ func ClusterTemplates(templatesList []*Template, options *protocols.ExecutorOpti
finalTemplatesList = append(finalTemplatesList, cluster...)
}
}
return finalTemplatesList, clusterCount
return finalTemplatesList, clusterCount, clusterMappings
}

// ClusterExecuter executes a group of requests for a protocol for a clustered
Expand Down
40 changes: 40 additions & 0 deletions pkg/templates/types/cluster_mappings.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package types

// ClusterMappingsMap wraps cluster ID to template IDs mapping
type ClusterMappingsMap struct {
Map map[string][]string
}

// NewClusterMappingsMap creates a new ClusterMappingsMap from an existing map
func NewClusterMappingsMap(m map[string][]string) *ClusterMappingsMap {
return &ClusterMappingsMap{Map: m}
}

// Get returns the template IDs for a given cluster ID, or nil, false if the receiver or Map is nil
func (c *ClusterMappingsMap) Get(clusterID string) ([]string, bool) {
if c == nil || c.Map == nil {
return nil, false
}
v, ok := c.Map[clusterID]
return v, ok
}

// GetAll returns a copy of the entire map, or an empty map if the receiver or Map is nil
func (c *ClusterMappingsMap) GetAll() map[string][]string {
if c == nil || c.Map == nil {
return make(map[string][]string)
}
result := make(map[string][]string, len(c.Map))
for k, v := range c.Map {
result[k] = append([]string{}, v...)
}
return result
}

// Copy returns a deep copy of the ClusterMappingsMap, or nil if the receiver is nil
func (c *ClusterMappingsMap) Copy() *ClusterMappingsMap {
if c == nil {
return nil
}
return NewClusterMappingsMap(c.GetAll())
}
2 changes: 1 addition & 1 deletion pkg/templates/workflows.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr
workflowTemplates = append(workflowTemplates, template)
}

finalTemplates, _ := ClusterTemplates(workflowTemplates, options.Copy())
finalTemplates, _, _ := ClusterTemplates(workflowTemplates, options.Copy())
for _, template := range finalTemplates {
workflow.Executers = append(workflow.Executers, &workflows.ProtocolExecuterPair{
Executer: template.Executer,
Expand Down
Loading