Merge pull request #3 from avithe-great/main

feat: Update the output json of the API scanning

Author: avithe-great

config/default.yaml

@@ -16,3 +16,5 @@ openAPISpec: <urlOrPath> # Either filepath or URL
 
 exporter:
   jsonReportFilePath: report.json
+
+scanName: default-openapi-spec-scan  

internal/apispec/oas.go

@@ -5,7 +5,7 @@ package apispec
 
 import (
 	"github.com/pb33f/libopenapi"
-	"github.com/pb33f/libopenapi/datamodel/high/v3"
+	v3 "github.com/pb33f/libopenapi/datamodel/high/v3"
 )
 
 func BuildOASV3Model(specBytes []byte) (*libopenapi.DocumentModel[v3.Document], error) {

internal/config/config.go

@@ -6,6 +6,7 @@ package config
 import (
 	"encoding/json"
 	"fmt"
+	"time"
 
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
@@ -36,6 +37,7 @@ type Configuration struct {
 	Environment Environment `json:"environment"`
 	OpenAPISpec string      `json:"openAPISpec"`
 	Exporter    Exporter    `json:"exporter,omitempty"`
+	ScanName    string      `json:"scanName"`
 }
 
 func (c *Configuration) validate() error {
@@ -59,10 +61,6 @@ func (c *Configuration) validate() error {
 		return fmt.Errorf("configuration does not contain a valid OpenAPI Specification filepath or URL")
 	}
 
-	if c.Environment.ClusterId == 0 {
-		return fmt.Errorf("please provide a valid cluster ID")
-	}
-
 	if c.Exporter.JsonReportFilePath == "" {
 		return fmt.Errorf("configuration does not contain a valid JSON reports file path")
 	}
@@ -90,6 +88,11 @@ func New(configFilePath string, logger *zap.SugaredLogger) (Configuration, error
 		logger.Warn("using default JSON report file path: ", defaultJSONReportFilePath)
 	}
 
+	if config.ScanName == "" {
+		config.ScanName = fmt.Sprintf("openapi-scan-%s", time.Now().Format("20060102-150405"))
+		logger.Infof("scanName not provided, using generated name: %s", config.ScanName)
+	}
+
 	if err := config.validate(); err != nil {
 		return Configuration{}, err
 	}

internal/core/apievent.go

@@ -11,11 +11,14 @@ import (
 )
 
 func (m *Manager) findDocuments(collectionName string, clusterId int) (*hashset.Set, error) {
-	// Todo: Process in batch of maybe 2000
 	filter := bson.D{
-		bson.E{Key: "operation", Value: "Api"},
-		bson.E{Key: "cluster_id", Value: clusterId},
+		{Key: "operation", Value: "Api"},
 	}
+
+	if clusterId != 0 {
+		filter = append(filter, bson.E{Key: "cluster_id", Value: clusterId})
+	}
+
 	projection := bson.D{
 		{Key: "_id", Value: 0},
 		{Key: "cluster_name", Value: 1},
@@ -28,7 +31,7 @@ func (m *Manager) findDocuments(collectionName string, clusterId int) (*hashset.
 
 	cursor, err := m.DBHandler.Database.
 		Collection(collectionName).
-		Find(m.Ctx, &filter, &options.FindOptions{
+		Find(m.Ctx, filter, &options.FindOptions{
 			Projection: &projection,
 		})
 	if err != nil {
@@ -52,6 +55,7 @@ func (m *Manager) findDocuments(collectionName string, clusterId int) (*hashset.
 		if responseCode == nil {
 			continue
 		}
+
 		apiEvents.Add(apievent.ApiEvent{
 			ClusterName:   document["cluster_name"].(string),
 			ServiceName:   document["api_event"].(bson.M)["http"].(bson.M)["request"].(bson.M)["headers"].(bson.M)[":authority"].(string),
@@ -61,8 +65,13 @@ func (m *Manager) findDocuments(collectionName string, clusterId int) (*hashset.
 			Occurrences:   int(document["api_event"].(bson.M)["count"].(int32)),
 		})
 	}
+
 	if apiEvents.Size() == 0 {
-		m.Logger.Warnf("no documents found in `%s` collection of clusterID: `%d`", collectionName, clusterId)
+		clusterInfo := fmt.Sprintf("clusterID: `%d`", clusterId)
+		if clusterId == 0 {
+			clusterInfo = "all clusters"
+		}
+		m.Logger.Warnf("no documents found in `%s` collection for %s", collectionName, clusterInfo)
 		return nil, nil
 	}
 

internal/core/core.go

@@ -68,7 +68,7 @@ func Run(ctx context.Context, configFilePath string) {
 
 	shadowApis, zombieApis := mgr.findShadowAndZombieApi(trie, events, model)
 	orphanApis := mgr.findOrphanApi(events, model)
-	if err := mgr.exportJsonReport(mgr.Cfg.Exporter.JsonReportFilePath, shadowApis, zombieApis, orphanApis, model.Model.Info, model.Index.GetConfig().SpecInfo.Version); err != nil {
+	if err := mgr.exportJsonReport(mgr.Cfg.Exporter.JsonReportFilePath, shadowApis, zombieApis, orphanApis); err != nil {
 		mgr.Logger.Error(err)
 		return
 	}

internal/core/discovery.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/emirpasic/gods/sets/hashset"
 	"github.com/pb33f/libopenapi"
-	"github.com/pb33f/libopenapi/datamodel/high/v3"
+	v3 "github.com/pb33f/libopenapi/datamodel/high/v3"
 
 	"github.com/5gsec/api-speculator/internal/apievent"
 	"github.com/5gsec/api-speculator/internal/apispec"

internal/core/reporter.go

@@ -6,8 +6,6 @@ package core
 import (
 	"encoding/json"
 	"os"
-
-	"github.com/pb33f/libopenapi/datamodel/high/base"
 )
 
 type API struct {
@@ -19,26 +17,20 @@ type API struct {
 }
 
 type apiReport struct {
-	ClusterId   int    `json:"clusterId"`
-	TenantId    int    `json:"tenantId"`
-	SpecTitle   string `json:"specTitle"`
-	SpecVersion string `json:"specVersion"`
-	OASVersion  string `json:"oasVersion"`
-	ShadowAPIs  []API  `json:"shadowApis,omitempty"`
-	ZombieAPIs  []API  `json:"zombieApis,omitempty"`
-	OrphanAPIs  []API  `json:"orphanApis,omitempty"`
+	TenantId   int    `json:"tenantId"`
+	ScanName   string `json:"scan_name"`
+	ShadowAPIs []API  `json:"shadowApis,omitempty"`
+	ZombieAPIs []API  `json:"zombieApis,omitempty"`
+	OrphanAPIs []API  `json:"orphanApis,omitempty"`
 }
 
-func (m *Manager) exportJsonReport(reportFilePath string, shadowApis, zombieApis, orphanApis []API, specInfo *base.Info, openApiVersion string) error {
+func (m *Manager) exportJsonReport(reportFilePath string, shadowApis, zombieApis, orphanApis []API) error {
 	report := apiReport{
-		ClusterId:   m.Cfg.Environment.ClusterId,
-		TenantId:    m.Cfg.Environment.TenantId,
-		SpecTitle:   specInfo.Title,
-		SpecVersion: specInfo.Version,
-		OASVersion:  openApiVersion,
-		ShadowAPIs:  shadowApis,
-		ZombieAPIs:  zombieApis,
-		OrphanAPIs:  orphanApis,
+		TenantId:   m.Cfg.Environment.TenantId,
+		ScanName:   m.Cfg.ScanName,
+		ShadowAPIs: shadowApis,
+		ZombieAPIs: zombieApis,
+		OrphanAPIs: orphanApis,
 	}
 
 	f, err := os.OpenFile(reportFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)