Skip to content

Commit dbaa5df

Browse files
alan20854alan20854
authored
NIGH-154 Default Config (#42)
* add default detectors when nightfall config missing/invalid * fix autoimport spacing * move default logic to nightfall_config * set default max routines to 20, not actual max if input is missing/invalid * pr updates * move logic to set number of routines to nightfall_config * use const instead of hard-coded val * pr updates * rm outdated log statements
1 parent 792a7da commit dbaa5df

File tree

7 files changed

+117
-35
lines changed

7 files changed

+117
-35
lines changed

‎internal/clients/diffreviewer/circleci/circleci_service.go‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
"github.com/nightfallai/nightfall_code_scanner/internal/clients/gitdiff"
1616
"github.com/nightfallai/nightfall_code_scanner/internal/clients/logger"
1717
circlelogger "github.com/nightfallai/nightfall_code_scanner/internal/clients/logger/circle_logger"
18-
"github.com/nightfallai/nightfall_code_scanner/internal/clients/nightfall"
1918
"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/gitdiffintf"
2019
"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/githubintf"
2120
"github.com/nightfallai/nightfall_code_scanner/internal/nightfallconfig"
@@ -100,7 +99,7 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
10099
BaseSHA: beforeCommitSha,
101100
Head: s.PrDetails.CommitSha,
102101
}
103-
nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName)
102+
nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName, s.Logger)
104103
if err != nil {
105104
s.Logger.Error("Error getting Nightfall config file. Ensure you have a Nightfall config file located in the root of your repository at .nightfalldlp/config.json with at least one Detector enabled")
106105
return nil, err
@@ -110,17 +109,10 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
110109
s.Logger.Error(fmt.Sprintf("Error getting Nightfall API key. Ensure you have %s set in the Github secrets of the repo", NightfallAPIKeyEnvVar))
111110
return nil, errors.New("missing env var for nightfall api key")
112111
}
113-
114-
var maxNumberRoutines int
115-
if nightfallConfig.MaxNumberRoutines < nightfall.MaxConcurrentRoutinesCap {
116-
maxNumberRoutines = nightfallConfig.MaxNumberRoutines
117-
} else {
118-
maxNumberRoutines = nightfall.MaxConcurrentRoutinesCap
119-
}
120112
return &nightfallconfig.Config{
121113
NightfallAPIKey: nightfallAPIKey,
122114
NightfallDetectors: nightfallConfig.Detectors,
123-
NightfallMaxNumberRoutines: maxNumberRoutines,
115+
NightfallMaxNumberRoutines: nightfallConfig.MaxNumberRoutines,
124116
TokenExclusionList: nightfallConfig.TokenExclusionList,
125117
FileInclusionList: nightfallConfig.FileInclusionList,
126118
FileExclusionList: nightfallConfig.FileExclusionList,

‎internal/clients/diffreviewer/circleci/circleci_service_test.go‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ const testOwner = "alan20854"
137137
const testRepo = "TestRepo"
138138
const testPrUrl = "https://github.com/alan20854/CircleCiTest/pull/3"
139139
const testConfigFileName = "nightfall_test_config.json"
140+
const testEmptyConfigFileName = "nightfall_empty_test_config.json"
140141
const excludedCreditCardRegex = "4242-4242-4242-[0-9]{4}"
141142
const excludedApiToken = "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv"
142143
const excludedIPRegex = "^127\\."
@@ -187,7 +188,7 @@ func (c *circleCiTestSuite) TestLoadConfig() {
187188
}
188189

189190
nightfallConfig, err := tp.cs.LoadConfig(testConfigFileName)
190-
c.NoError(err, "Error in LoadConfig")
191+
c.NoError(err, "Unexpected error in LoadConfig")
191192
c.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
192193
}
193194

@@ -211,6 +212,34 @@ func (c *circleCiTestSuite) TestLoadConfigMissingApiKey() {
211212
)
212213
}
213214

215+
func (c *circleCiTestSuite) TestLoadEmptyConfig() {
216+
tp := c.initTestParams()
217+
apiKey := "api-key"
218+
apiDetector := nightfallAPI.API_KEY
219+
cryptoDetector := nightfallAPI.CRYPTOGRAPHIC_TOKEN
220+
workspace, err := os.Getwd()
221+
c.NoError(err, "Error getting workspace")
222+
workspacePath := path.Join(workspace, "../../../../test/data")
223+
os.Setenv(WorkspacePathEnvVar, workspacePath)
224+
os.Setenv(CircleCurrentCommitShaEnvVar, commitSha)
225+
os.Setenv(CircleBeforeCommitEnvVar, prevCommitSha)
226+
os.Setenv(CircleBranchEnvVar, testBranch)
227+
os.Setenv(CircleOwnerNameEnvVar, testOwner)
228+
os.Setenv(CircleRepoNameEnvVar, testRepo)
229+
os.Setenv(CirclePullRequestUrlEnvVar, testPrUrl)
230+
os.Setenv(NightfallAPIKeyEnvVar, apiKey)
231+
232+
expectedNightfallConfig := &nightfallconfig.Config{
233+
NightfallAPIKey: apiKey,
234+
NightfallDetectors: []*nightfallAPI.Detector{&apiDetector, &cryptoDetector},
235+
NightfallMaxNumberRoutines: nightfallconfig.DefaultMaxNumberRoutines,
236+
}
237+
238+
nightfallConfig, err := tp.cs.LoadConfig(testEmptyConfigFileName)
239+
c.NoError(err, "Unexpected error in LoadConfig")
240+
c.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
241+
}
242+
214243
func (c *circleCiTestSuite) TestGetDiff() {
215244
tp := c.initTestParams()
216245
ctrl := gomock.NewController(c.T())

‎internal/clients/diffreviewer/github/github_service.go‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,12 @@ import (
99
"os"
1010
"strings"
1111

12-
"github.com/nightfallai/nightfall_code_scanner/internal/clients/diffreviewer/diffutils"
13-
1412
"github.com/google/go-github/v31/github"
1513
"github.com/nightfallai/nightfall_code_scanner/internal/clients/diffreviewer"
14+
"github.com/nightfallai/nightfall_code_scanner/internal/clients/diffreviewer/diffutils"
1615
"github.com/nightfallai/nightfall_code_scanner/internal/clients/gitdiff"
1716
"github.com/nightfallai/nightfall_code_scanner/internal/clients/logger"
1817
githublogger "github.com/nightfallai/nightfall_code_scanner/internal/clients/logger/github_logger"
19-
"github.com/nightfallai/nightfall_code_scanner/internal/clients/nightfall"
2018
"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/gitdiffintf"
2119
"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/githubintf"
2220
"github.com/nightfallai/nightfall_code_scanner/internal/nightfallconfig"
@@ -189,7 +187,7 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
189187
BaseSHA: event.Before,
190188
Head: s.CheckRequest.SHA,
191189
}
192-
nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName)
190+
nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName, s.Logger)
193191
if err != nil {
194192
s.Logger.Error("Error getting Nightfall config file. Ensure you have a Nightfall config file located in the root of your repository at .nightfalldlp/config.json with at least one Detector enabled")
195193
return nil, err
@@ -199,16 +197,10 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
199197
s.Logger.Error(fmt.Sprintf("Error getting Nightfall API key. Ensure you have %s set in the Github secrets of the repo", NightfallAPIKeyEnvVar))
200198
return nil, errors.New("Missing env var for nightfall api key")
201199
}
202-
var maxNumberRoutines int
203-
if nightfallConfig.MaxNumberRoutines < nightfall.MaxConcurrentRoutinesCap {
204-
maxNumberRoutines = nightfallConfig.MaxNumberRoutines
205-
} else {
206-
maxNumberRoutines = nightfall.MaxConcurrentRoutinesCap
207-
}
208200
return &nightfallconfig.Config{
209201
NightfallAPIKey: nightfallAPIKey,
210202
NightfallDetectors: nightfallConfig.Detectors,
211-
NightfallMaxNumberRoutines: maxNumberRoutines,
203+
NightfallMaxNumberRoutines: nightfallConfig.MaxNumberRoutines,
212204
TokenExclusionList: nightfallConfig.TokenExclusionList,
213205
FileInclusionList: nightfallConfig.FileInclusionList,
214206
FileExclusionList: nightfallConfig.FileExclusionList,

‎internal/clients/diffreviewer/github/github_service_test.go‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ func (g *githubTestSuite) initTestParams() *testParams {
138138
}
139139

140140
const testConfigFileName = "nightfall_test_config.json"
141+
const testEmptyConfigFileName = "nightfall_test_empty_config.json"
141142
const excludedCreditCardRegex = "4242-4242-4242-[0-9]{4}"
142143
const excludedApiToken = "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv"
143144
const excludedIPRegex = "^127\\."
@@ -189,9 +190,44 @@ func (g *githubTestSuite) TestLoadConfig() {
189190
}
190191

191192
nightfallConfig, err := tp.gc.LoadConfig(testConfigFileName)
192-
g.NoError(err, "Error in LoadConfig")
193+
g.NoError(err, "Unexpected error in LoadConfig")
193194
g.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
194-
g.Equal(expectedGithubCheckRequest, tp.gc.CheckRequest, "Incorrect nightfall config")
195+
g.Equal(expectedGithubCheckRequest, tp.gc.CheckRequest, "Incorrect github check request")
196+
}
197+
198+
func (g *githubTestSuite) TestLoadEmptyConfig() {
199+
tp := g.initTestParams()
200+
apiKey := "api-key"
201+
sha := "1234"
202+
owner := "nightfallai"
203+
repo := "testRepo"
204+
apiDetector := nightfallAPI.API_KEY
205+
cryptoDetector := nightfallAPI.CRYPTOGRAPHIC_TOKEN
206+
pullRequest := 1
207+
workspace, err := os.Getwd()
208+
g.NoError(err, "Error getting workspace")
209+
workspacePath := path.Join(workspace, "../../../../test/data")
210+
eventPath := path.Join(workspace, "../../../../test/data/github_action_event.json")
211+
os.Setenv(githubservice.WorkspacePathEnvVar, workspacePath)
212+
os.Setenv(githubservice.EventPathEnvVar, eventPath)
213+
os.Setenv(githubservice.NightfallAPIKeyEnvVar, apiKey)
214+
215+
expectedNightfallConfig := &nightfallconfig.Config{
216+
NightfallAPIKey: apiKey,
217+
NightfallDetectors: []*nightfallAPI.Detector{&apiDetector, &cryptoDetector},
218+
NightfallMaxNumberRoutines: nightfallconfig.DefaultMaxNumberRoutines,
219+
}
220+
expectedGithubCheckRequest := &githubservice.CheckRequest{
221+
Owner: owner,
222+
Repo: repo,
223+
SHA: sha,
224+
PullRequest: pullRequest,
225+
}
226+
227+
nightfallConfig, err := tp.gc.LoadConfig(testEmptyConfigFileName)
228+
g.NoError(err, "Unexpected error in LoadConfig")
229+
g.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
230+
g.Equal(expectedGithubCheckRequest, tp.gc.CheckRequest, "Incorrect github check request")
195231
}
196232

197233
func (g *githubTestSuite) TestGetDiff() {

‎internal/clients/nightfall/nightfall.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ const (
3131
maxItemsForAPIReq = 479
3232
// timeout for the total time spent sending scan requests and receiving responses for a diff
3333
defaultTimeout = time.Minute * 20
34-
// maximum number of routines (scan request + response) running at once
35-
MaxConcurrentRoutinesCap = 50
3634
// maximum attempts to Nightfall API upon receiving 429 Too Many Requests before failing
3735
MaxScanAttempts = 5
3836
// initial delay before re-attempting scan request

‎internal/nightfallconfig/nightfall_config.go‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,23 @@ package nightfallconfig
33
import (
44
"encoding/json"
55
"errors"
6+
"fmt"
67
"io/ioutil"
78
"os"
89
"path"
910

11+
"github.com/nightfallai/nightfall_code_scanner/internal/clients/logger"
1012
nightfallAPI "github.com/nightfallai/nightfall_go_client/generated"
1113
)
1214

13-
const defaultMaxNumberRoutines = 30
15+
// maximum number of routines (scan request + response) running at once
16+
const MaxConcurrentRoutinesCap = 50
17+
const DefaultMaxNumberRoutines = 20
1418
const nightfallConfigFilename = ".nightfalldlp/config.json"
19+
const defaultDetectorsInfoMessage = "Using default detectors (API_KEY and CRYTOGRAPHIC_TOKEN)"
20+
21+
var apiKeyDetector = nightfallAPI.API_KEY
22+
var cryptoKeyDetector = nightfallAPI.CRYPTOGRAPHIC_TOKEN
1523

1624
// NightfallConfigFileStructure struct representation of nightfall config file
1725
type NightfallConfigFileStructure struct {
@@ -32,16 +40,24 @@ type Config struct {
3240
FileExclusionList []string
3341
}
3442

35-
// GetNightfallConfigFile loads nightfall config from file
36-
func GetNightfallConfigFile(workspacePath, fileName string) (*NightfallConfigFileStructure, error) {
43+
// GetNightfallConfigFile loads nightfall config from file, returns default if missing/invalid
44+
func GetNightfallConfigFile(workspacePath, fileName string, logger logger.Logger) (*NightfallConfigFileStructure, error) {
45+
defaultNightfallConfig := &NightfallConfigFileStructure{
46+
Detectors: []*nightfallAPI.Detector{&apiKeyDetector, &cryptoKeyDetector},
47+
MaxNumberRoutines: DefaultMaxNumberRoutines,
48+
}
3749
nightfallConfigFile, err := os.Open(path.Join(workspacePath, fileName))
3850
if err != nil {
39-
return nil, err
51+
logger.Warning(fmt.Sprintf("Error opening nightfall config: %s", err.Error()))
52+
logger.Info(defaultDetectorsInfoMessage)
53+
return defaultNightfallConfig, nil
4054
}
4155
defer nightfallConfigFile.Close()
4256
byteValue, err := ioutil.ReadAll(nightfallConfigFile)
4357
if err != nil {
44-
return nil, err
58+
logger.Warning(fmt.Sprintf("Error reading nightfall config: %s", err.Error()))
59+
logger.Info(defaultDetectorsInfoMessage)
60+
return defaultNightfallConfig, nil
4561
}
4662
var nightfallConfig NightfallConfigFileStructure
4763
err = json.Unmarshal(byteValue, &nightfallConfig)
@@ -51,8 +67,10 @@ func GetNightfallConfigFile(workspacePath, fileName string) (*NightfallConfigFil
5167
if len(nightfallConfig.Detectors) < 1 {
5268
return nil, errors.New("Nightfall config file is missing detectors")
5369
}
54-
if nightfallConfig.MaxNumberRoutines == 0 {
55-
nightfallConfig.MaxNumberRoutines = defaultMaxNumberRoutines
70+
if nightfallConfig.MaxNumberRoutines <= 0 {
71+
nightfallConfig.MaxNumberRoutines = DefaultMaxNumberRoutines
72+
} else if nightfallConfig.MaxNumberRoutines > MaxConcurrentRoutinesCap {
73+
nightfallConfig.MaxNumberRoutines = MaxConcurrentRoutinesCap
5674
}
5775
nightfallConfig.FileExclusionList = append(nightfallConfig.FileExclusionList, nightfallConfigFilename)
5876
return &nightfallConfig, nil

‎internal/nightfallconfig/nightfall_config_test.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@ import (
55
"path"
66
"testing"
77

8+
githublogger "github.com/nightfallai/nightfall_code_scanner/internal/clients/logger/github_logger"
89
"github.com/nightfallai/nightfall_code_scanner/internal/nightfallconfig"
910
nightfallAPI "github.com/nightfallai/nightfall_go_client/generated"
1011
"github.com/stretchr/testify/assert"
1112
)
1213

1314
const testFileName = "nightfall_test_config.json"
15+
const testMissingFileName = "nightfall_test_missing_config.json"
1416
const excludedCreditCardRegex = "4242-4242-4242-[0-9]{4}"
1517
const excludedApiToken = "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv"
1618
const excludedIPRegex = "^127\\."
@@ -29,7 +31,22 @@ func TestGetNightfallConfig(t *testing.T) {
2931
FileInclusionList: []string{"*"},
3032
FileExclusionList: []string{".nightfalldlp/config.json"},
3133
}
32-
actualConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, testFileName)
33-
assert.NoError(t, err, "Unexpected error when GetNightfallConfig")
34+
actualConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, testFileName, nil)
35+
assert.NoError(t, err, "Unexpected error in test GetNightfallConfig")
36+
assert.Equal(t, expectedConfig, actualConfig, "Incorrect nightfall config")
37+
}
38+
39+
func TestGetNightfallConfigMissingConfigFile(t *testing.T) {
40+
apiDetector := nightfallAPI.API_KEY
41+
cryptoDetector := nightfallAPI.CRYPTOGRAPHIC_TOKEN
42+
workspaceConfig, err := os.Getwd()
43+
assert.NoError(t, err, "Unexpected error when getting current directory")
44+
workspacePath := path.Join(workspaceConfig, "../../test/data")
45+
expectedConfig := &nightfallconfig.NightfallConfigFileStructure{
46+
Detectors: []*nightfallAPI.Detector{&apiDetector, &cryptoDetector},
47+
MaxNumberRoutines: nightfallconfig.DefaultMaxNumberRoutines,
48+
}
49+
actualConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, testMissingFileName, githublogger.NewDefaultGithubLogger())
50+
assert.NoError(t, err, "Unexpected error in test GetNightfallConfigMissingConfigFile")
3451
assert.Equal(t, expectedConfig, actualConfig, "Incorrect nightfall config")
3552
}

0 commit comments

Comments
 (0)