Merge pull request #5 from avithe-great/feat-orphan-api

feat(Orphan APIs): update orphan API detection logic

Author: avithe-great

cmd/root.go

@@ -27,12 +27,14 @@ func init() {
 
 var RootCmd = &cobra.Command{
 	Use:   "speculator",
-	Short: "speculator is a utility that identifies shadow and zombie APIs by analyzing API traffic against provided API specifications",
-	Long: `speculator helps you to secure your APIs by identifying shadow and zombie APIs.
+	Short: "speculator is a utility that identifies shadow, orphan, and zombie APIs by analyzing API traffic against provided API specifications",
+
+	Long: `speculator helps you secure your APIs by identifying shadow, orphan, and zombie APIs.
 
 By analyzing API traffic in conjunction with your API specifications (e.g., OpenAPI, Swagger), speculator can detect:
   * Shadow APIs: Endpoints that are implemented and functional but not documented in your API specification.
-  * Zombie APIs: Endpoints that are deprecated or abandoned in your API specification but they are still in use.
+  * Zombie APIs: Endpoints that are deprecated or abandoned in your API specification but are still in use.
+  * Orphan APIs: Endpoints that are defined in your API specification but are never invoked in the observed traffic.
 `,
 	//Long: `A Utility to identify Shadow and Zombie APIs provided API Specification`,
 	Run: func(cmd *cobra.Command, args []string) {

go.mod

@@ -8,13 +8,15 @@ require (
 	github.com/pb33f/libopenapi v0.21.9
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1
+	github.com/stretchr/testify v1.10.0
 	go.mongodb.org/mongo-driver v1.17.3
 	go.uber.org/zap v1.27.0
 )
 
 require (
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
@@ -23,6 +25,7 @@ require (
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/sagikazarmark/locafero v0.9.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect

internal/apispec/pathparam.go

@@ -14,12 +14,26 @@ import (
 
 var digitCheck = regexp.MustCompile(`^[0-9]+$`)
 
-func UnifyParameterizedPathIfApplicable(path string) string {
+// UnifyParameterizedPathIfApplicable normalizes a path by replacing dynamic segments with {paramN}.
+// If isSpec = true, also treats existing {param} segments in OpenAPI specs as parameters.
+func UnifyParameterizedPathIfApplicable(path string, isSpec bool) string {
+	if path == "" {
+		return ""
+	}
+	if path == "/" {
+		return "/"
+	}
+
+	pathParts := strings.Split(strings.Trim(path, "/"), "/")
 	var parameterizedPathParts []string
 	paramCount := 0
-	pathParts := strings.Split(path, "/")
 
 	for _, part := range pathParts {
+		if isSpec && strings.HasPrefix(part, "{") && strings.HasSuffix(part, "}") {
+			parameterizedPathParts = append(parameterizedPathParts, part)
+			continue
+		}
+
 		if isSuspectPathParam(part) {
 			paramCount++
 			paramName := fmt.Sprintf("param%v", paramCount)
@@ -28,36 +42,19 @@ func UnifyParameterizedPathIfApplicable(path string) string {
 			parameterizedPathParts = append(parameterizedPathParts, part)
 		}
 	}
-	return strings.Join(parameterizedPathParts, "/")
-}
-
-func isSuspectPathParam(pathPart string) bool {
-	if isNumber(pathPart) {
-		return true
-	}
-	if isUUID(pathPart) {
-		return true
-	}
-	if isMixed(pathPart) {
-		return true
-	}
-	if isString(pathPart) {
-		return true
-	}
-	return false
+	return "/" + strings.Join(parameterizedPathParts, "/")
 }
 
-func isString(pathPart string) bool {
-	return strings.Contains(pathPart, "{") &&
-		strings.Contains(pathPart, "}")
+func isSuspectPathParam(part string) bool {
+	return isNumber(part) || isUUID(part) || isMixed(part)
 }
 
-func isNumber(pathPart string) bool {
-	return digitCheck.MatchString(pathPart)
+func isNumber(s string) bool {
+	return digitCheck.MatchString(s)
 }
 
-func isUUID(pathPart string) bool {
-	_, err := uuid.FromString(pathPart)
+func isUUID(s string) bool {
+	_, err := uuid.FromString(s)
 	return err == nil
 }
 

internal/apispec/pathparam_test.go

@@ -0,0 +1,84 @@
+package apispec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnifyParameterizedPathIfApplicable(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		isSpec   bool
+		expected string
+	}{
+		{
+			name:     "static path - no change",
+			input:    "/users/list",
+			isSpec:   false,
+			expected: "/users/list",
+		},
+		{
+			name:     "numeric ID in path",
+			input:    "/users/123",
+			isSpec:   false,
+			expected: "/users/{param1}",
+		},
+		{
+			name:     "UUID in path",
+			input:    "/orders/550e8400-e29b-41d4-a716-446655440000",
+			isSpec:   false,
+			expected: "/orders/{param1}",
+		},
+		{
+			name:     "mixed alphanumeric long string",
+			input:    "/data/abc12345xyz",
+			isSpec:   false,
+			expected: "/data/{param1}",
+		},
+		{
+			name:     "short alphanumeric (not treated as param)",
+			input:    "/data/ab12",
+			isSpec:   false,
+			expected: "/data/ab12",
+		},
+		{
+			name:     "multiple parameters in path",
+			input:    "/users/123/orders/550e8400-e29b-41d4-a716-446655440000",
+			isSpec:   false,
+			expected: "/users/{param1}/orders/{param2}",
+		},
+		{
+			name:     "already parameterized spec path",
+			input:    "/users/{userId}",
+			isSpec:   true,
+			expected: "/users/{userId}",
+		},
+		{
+			name:     "spec path with multiple params",
+			input:    "/users/{userId}/orders/{orderId}",
+			isSpec:   true,
+			expected: "/users/{userId}/orders/{orderId}",
+		},
+		{
+			name:     "root path",
+			input:    "/",
+			isSpec:   false,
+			expected: "/",
+		},
+		{
+			name:     "empty path",
+			input:    "",
+			isSpec:   false,
+			expected: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := UnifyParameterizedPathIfApplicable(tt.input, tt.isSpec)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}

internal/apispec/query.go

@@ -21,19 +21,13 @@ func ExtractQueryAndParams(path string) (string, url.Values) {
 	return query, values
 }
 
-func GetPathAndQuery(fullPath string) (path, query string) {
-	// Example: "/example-path?param=value" returns "/example-path", "param=value"
-	index := strings.IndexByte(fullPath, '?')
-	if index == -1 {
-		return fullPath, ""
+// GetPathAndQuery splits a URL into path and query components.
+func GetPathAndQuery(fullPath string) (string, string) {
+	if idx := strings.IndexByte(fullPath, '?'); idx != -1 {
+		if idx == len(fullPath)-1 {
+			return fullPath[:idx], ""
+		}
+		return fullPath[:idx], fullPath[idx+1:]
 	}
-
-	// Example: "/path?" returns "/path?", ""
-	if index == (len(fullPath) - 1) {
-		return fullPath, ""
-	}
-
-	path = fullPath[:index]
-	query = fullPath[index+1:]
-	return
+	return fullPath, ""
 }

internal/apispec/query_test.go

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright 2024 Authors of API-Speculator
+
+package apispec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetPathAndQuery(t *testing.T) {
+	tests := []struct {
+		input     string
+		wantPath  string
+		wantQuery string
+	}{
+		{"/api/v1/users?id=123", "/api/v1/users", "id=123"},
+		{"/api/v1/users?", "/api/v1/users", ""},
+		{"/api/v1/users", "/api/v1/users", ""},
+	}
+
+	for _, tt := range tests {
+		path, query := GetPathAndQuery(tt.input)
+		assert.Equal(t, tt.wantPath, path)
+		assert.Equal(t, tt.wantQuery, query)
+	}
+}

internal/core/discovery.go

@@ -88,7 +88,7 @@ func (m *Manager) findOrphanApi(events *hashset.Set, model *libopenapi.DocumentM
 		}
 
 		requestPath, _ := apispec.GetPathAndQuery(event.RequestPath)
-		requestPath = apispec.UnifyParameterizedPathIfApplicable(requestPath)
+		requestPath = apispec.UnifyParameterizedPathIfApplicable(requestPath, false)
 		requestMethod := strings.ToUpper(event.RequestMethod)
 		key := fmt.Sprintf("%v/%v", requestMethod, requestPath)
 
@@ -99,7 +99,7 @@ func (m *Manager) findOrphanApi(events *hashset.Set, model *libopenapi.DocumentM
 
 	for pathItems := model.Model.Paths.PathItems.First(); pathItems != nil; pathItems = pathItems.Next() {
 		for operations := pathItems.Value().GetOperations().First(); operations != nil; operations = operations.Next() {
-			requestPath := apispec.UnifyParameterizedPathIfApplicable(pathItems.Key())
+			requestPath := apispec.UnifyParameterizedPathIfApplicable(pathItems.Key(), true)
 			requestMethod := strings.ToUpper(operations.Key())
 			key := fmt.Sprintf("%v/%v", requestMethod, requestPath)