Skip to content

Commit 1c2ef2f

Browse files
committed
Merge branch 'main' into enable_acs_deploy
2 parents 9caf4f0 + 904ed46 commit 1c2ef2f

File tree

26 files changed

+507
-218
lines changed

26 files changed

+507
-218
lines changed
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
# The default branch of security_content should always be correct.
2+
# As such, we should use it in our test workflow, here, to ensure
3+
# that contentctl is also correct and does not throw unexpected errors.
4+
5+
# We should remember that if contentctl introduces NEW validations that have
6+
# note yet been fixed in security_content, we may see this workflow fail.
7+
name: test_against_escu
8+
on:
9+
push:
10+
pull_request:
11+
types: [opened, reopened]
12+
schedule:
13+
- cron: "44 4 * * *"
14+
15+
jobs:
16+
smoketest_escu:
17+
strategy:
18+
fail-fast: false
19+
matrix:
20+
python_version: ["3.11", "3.12"]
21+
operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14"]
22+
#operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"]
23+
24+
25+
runs-on: ${{ matrix.operating_system }}
26+
steps:
27+
# Checkout the current branch of contentctl repo
28+
- name: Checkout repo
29+
uses: actions/checkout@v4
30+
31+
# Checkout the develop (default) branch of security_content
32+
- name: Checkout repo
33+
uses: actions/checkout@v4
34+
with:
35+
path: security_content
36+
repository: splunk/security_content
37+
38+
#Install the given version of Python we will test against
39+
- name: Install Required Python Version
40+
uses: actions/setup-python@v5
41+
with:
42+
python-version: ${{ matrix.python_version }}
43+
architecture: "x64"
44+
45+
- name: Install Poetry
46+
run:
47+
python -m pip install poetry
48+
49+
- name: Install contentctl and activate the shell
50+
run: |
51+
poetry install --no-interaction
52+
53+
54+
- name: Clone the AtomicRedTeam Repo (for extended validation)
55+
run: |
56+
cd security_content
57+
git clone --depth 1 https://github.com/redcanaryco/atomic-red-team
58+
59+
60+
# We do not separately run validate and build
61+
# since a build ALSO performs a validate
62+
- name: Run contentctl build
63+
run: |
64+
cd security_content
65+
poetry run contentctl build --enrichments
66+
67+
# Do not run a test - it will take far too long!
68+
# Do not upload any artifacts
69+

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ apps*
1010
test_results*
1111
attack_data*
1212
security_content/
13+
contentctl.yml
1314

1415
# Byte-compiled / optimized / DLL files
1516
__pycache__/

contentctl/actions/deploy_acs.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,4 @@ def execute(self, config: deploy_acs, appinspect_token:str) -> None:
4949
formatted_error_text = pprint.pformat(error_text)
5050
raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}")
5151

52-
print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")
53-
54-
55-
52+
print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")

contentctl/actions/new_content.py

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,22 @@ def buildDetection(self)->dict[str,Any]:
1919
answers = questionary.prompt(questions)
2020
answers.update(answers)
2121
answers['name'] = answers['detection_name']
22+
del answers['detection_name']
2223
answers['id'] = str(uuid.uuid4())
2324
answers['version'] = 1
2425
answers['date'] = datetime.today().strftime('%Y-%m-%d')
2526
answers['author'] = answers['detection_author']
26-
answers['data_source'] = answers['data_source']
27+
del answers['detection_author']
28+
answers['data_sources'] = answers['data_source']
29+
del answers['data_source']
2730
answers['type'] = answers['detection_type']
31+
del answers['detection_type']
2832
answers['status'] = "production" #start everything as production since that's what we INTEND the content to become
2933
answers['description'] = 'UPDATE_DESCRIPTION'
3034
file_name = answers['name'].replace(' ', '_').replace('-','_').replace('.','_').replace('/','_').lower()
35+
answers['kind'] = answers['detection_kind']
3136
answers['search'] = answers['detection_search'] + ' | `' + file_name + '_filter`'
37+
del answers['detection_search']
3238
answers['how_to_implement'] = 'UPDATE_HOW_TO_IMPLEMENT'
3339
answers['known_false_positives'] = 'UPDATE_KNOWN_FALSE_POSITIVES'
3440
answers['references'] = ['REFERENCE']
@@ -44,6 +50,7 @@ def buildDetection(self)->dict[str,Any]:
4450
answers['tags']['required_fields'] = ['UPDATE']
4551
answers['tags']['risk_score'] = 'UPDATE (impact * confidence)/100'
4652
answers['tags']['security_domain'] = answers['security_domain']
53+
del answers["security_domain"]
4754
answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE']
4855

4956
#generate the tests section
@@ -52,45 +59,49 @@ def buildDetection(self)->dict[str,Any]:
5259
'name': "True Positive Test",
5360
'attack_data': [
5461
{
55-
'data': "Enter URL for Dataset Here. This may also be a relative or absolute path on your local system for testing.",
62+
'data': "https://github.com/splunk/contentctl/wiki",
5663
"sourcetype": "UPDATE SOURCETYPE",
5764
"source": "UPDATE SOURCE"
5865
}
5966
]
6067
}
6168
]
69+
del answers["mitre_attack_ids"]
6270
return answers
6371

6472
def buildStory(self)->dict[str,Any]:
6573
questions = NewContentQuestions.get_questions_story()
6674
answers = questionary.prompt(questions)
6775
answers['name'] = answers['story_name']
76+
del answers['story_name']
6877
answers['id'] = str(uuid.uuid4())
6978
answers['version'] = 1
7079
answers['date'] = datetime.today().strftime('%Y-%m-%d')
7180
answers['author'] = answers['story_author']
81+
del answers['story_author']
7282
answers['description'] = 'UPDATE_DESCRIPTION'
7383
answers['narrative'] = 'UPDATE_NARRATIVE'
7484
answers['references'] = []
7585
answers['tags'] = dict()
76-
answers['tags']['analytic_story'] = answers['name']
7786
answers['tags']['category'] = answers['category']
87+
del answers['category']
7888
answers['tags']['product'] = ['Splunk Enterprise','Splunk Enterprise Security','Splunk Cloud']
7989
answers['tags']['usecase'] = answers['usecase']
90+
del answers['usecase']
8091
answers['tags']['cve'] = ['UPDATE WITH CVE(S) IF APPLICABLE']
8192
return answers
8293

8394

8495
def execute(self, input_dto: new) -> None:
8596
if input_dto.type == NewContentType.detection:
8697
content_dict = self.buildDetection()
87-
subdirectory = pathlib.Path('detections') / content_dict.get('type')
98+
subdirectory = pathlib.Path('detections') / content_dict.pop('detection_kind')
8899
elif input_dto.type == NewContentType.story:
89100
content_dict = self.buildStory()
90101
subdirectory = pathlib.Path('stories')
91102
else:
92103
raise Exception(f"Unsupported new content type: [{input_dto.type}]")
93-
104+
94105
full_output_path = input_dto.path / subdirectory / SecurityContentObject_Abstract.contentNameToFileName(content_dict.get('name'))
95106
YmlWriter.writeYmlFile(str(full_output_path), content_dict)
96107

@@ -103,12 +114,12 @@ def writeObjectNewContent(self, object: dict, subdirectory_name: str, type: NewC
103114
#make sure the output folder exists for this detection
104115
output_folder.mkdir(exist_ok=True)
105116

106-
YmlWriter.writeYmlFile(file_path, object)
117+
YmlWriter.writeDetection(file_path, object)
107118
print("Successfully created detection " + file_path)
108119

109120
elif type == NewContentType.story:
110121
file_path = os.path.join(self.output_path, 'stories', self.convertNameToFileName(object['name'], object['tags']['product']))
111-
YmlWriter.writeYmlFile(file_path, object)
122+
YmlWriter.writeStory(file_path, object)
112123
print("Successfully created story " + file_path)
113124

114125
else:

contentctl/actions/validate.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ def execute(self, input_dto: validate) -> DirectorOutputDto:
2323
director_output_dto = DirectorOutputDto(AtomicTest.getAtomicTestsFromArtRepo(repo_path=input_dto.getAtomicRedTeamRepoPath(),
2424
enabled=input_dto.enrichments),
2525
AttackEnrichment.getAttackEnrichment(input_dto),
26+
CveEnrichment.getCveEnrichment(input_dto),
2627
[],[],[],[],[],[],[],[],[])
2728

2829

contentctl/api.py

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
from pathlib import Path
2+
from typing import Any, Union, Type
3+
from contentctl.input.yml_reader import YmlReader
4+
from contentctl.objects.config import test_common, test, test_servers
5+
from contentctl.objects.security_content_object import SecurityContentObject
6+
from contentctl.input.director import DirectorOutputDto
7+
8+
def config_from_file(path:Path=Path("contentctl.yml"), config: dict[str,Any]={},
9+
configType:Type[Union[test,test_servers]]=test)->test_common:
10+
11+
"""
12+
Fetch a configuration object that can be used for a number of different contentctl
13+
operations including validate, build, inspect, test, and test_servers. A file will
14+
be used as the basis for constructing the configuration.
15+
16+
Args:
17+
path (Path, optional): Relative or absolute path to a contentctl config file.
18+
Defaults to Path("contentctl.yml"), which is the default name and location (in the current directory)
19+
of the configuration files which are automatically generated for contentctl.
20+
config (dict[], optional): Dictionary of values to override values read from the YML
21+
path passed as the first argument. Defaults to {}, an empty dict meaning that nothing
22+
will be overwritten
23+
configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate.
24+
This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test.
25+
Returns:
26+
test_common: Returns a complete contentctl test_common configuration. Note that this configuration
27+
will have all applicable field for validate and build as well, but can also be used for easily
28+
construction a test or test_servers object.
29+
"""
30+
31+
try:
32+
yml_dict = YmlReader.load_file(path, add_fields=False)
33+
34+
35+
except Exception as e:
36+
raise Exception(f"Failed to load contentctl configuration from file '{path}': {str(e)}")
37+
38+
# Apply settings that have been overridden from the ones in the file
39+
try:
40+
yml_dict.update(config)
41+
except Exception as e:
42+
raise Exception(f"Failed updating dictionary of values read from file '{path}'"
43+
f" with the dictionary of arguments passed: {str(e)}")
44+
45+
# The function below will throw its own descriptive exception if it fails
46+
configObject = config_from_dict(yml_dict, configType=configType)
47+
48+
return configObject
49+
50+
51+
52+
53+
def config_from_dict(config: dict[str,Any]={},
54+
configType:Type[Union[test,test_servers]]=test)->test_common:
55+
"""
56+
Fetch a configuration object that can be used for a number of different contentctl
57+
operations including validate, build, inspect, test, and test_servers. A dict will
58+
be used as the basis for constructing the configuration.
59+
60+
Args:
61+
config (dict[str,Any],Optional): If a dictionary is not explicitly passed, then
62+
an empty dict will be used to create a configuration, if possible, from default
63+
values. Note that based on default values in the contentctl/objects/config.py
64+
file, this may raise an exception. If so, please set appropriate default values
65+
in the file above or supply those values via this argument.
66+
configType (Type[Union[test,test_servers]], optional): The Config Class to instantiate.
67+
This may be a test or test_servers object. Note that this is NOT an instance of the class. Defaults to test.
68+
Returns:
69+
test_common: Returns a complete contentctl test_common configuration. Note that this configuration
70+
will have all applicable field for validate and build as well, but can also be used for easily
71+
construction a test or test_servers object.
72+
"""
73+
try:
74+
test_object = configType.model_validate(config)
75+
except Exception as e:
76+
raise Exception(f"Failed to load contentctl configuration from dict:\n{str(e)}")
77+
78+
return test_object
79+
80+
81+
def update_config(config:Union[test,test_servers], **key_value_updates:dict[str,Any])->test_common:
82+
83+
"""Update any relevant keys in a config file with the specified values.
84+
Full validation will be performed after this update and descriptive errors
85+
will be produced
86+
87+
Args:
88+
config (test_common): A previously-constructed test_common object. This can be
89+
build using the configFromDict or configFromFile functions.
90+
key_value_updates (kwargs, optional): Additional keyword/argument pairs to update
91+
arbitrary fields in the configuration.
92+
93+
Returns:
94+
test_common: A validated object which has had the relevant fields updated.
95+
Note that descriptive Exceptions will be generated if updated values are either
96+
invalid (have the wrong type, or disallowed values) or you attempt to update
97+
fields that do not exist
98+
"""
99+
# Create a copy so we don't change the underlying model
100+
config_copy = config.model_copy(deep=True)
101+
102+
# Force validation of assignment since doing so via arbitrary dict can be error prone
103+
# Also, ensure that we do not try to add fields that are not part of the model
104+
config_copy.model_config.update({'validate_assignment': True, 'extra': 'forbid'})
105+
106+
107+
108+
# Collect any errors that may occur
109+
errors:list[Exception] = []
110+
111+
# We need to do this one by one because the extra:forbid argument does not appear to
112+
# be respected at this time.
113+
for key, value in key_value_updates.items():
114+
try:
115+
setattr(config_copy,key,value)
116+
except Exception as e:
117+
errors.append(e)
118+
if len(errors) > 0:
119+
errors_string = '\n'.join([str(e) for e in errors])
120+
raise Exception(f"Error(s) updaitng configuration:\n{errors_string}")
121+
122+
return config_copy
123+
124+
125+
126+
def content_to_dict(director:DirectorOutputDto)->dict[str,list[dict[str,Any]]]:
127+
output_dict:dict[str,list[dict[str,Any]]] = {}
128+
for contentType in ['detections','stories','baselines','investigations',
129+
'playbooks','macros','lookups','deployments','ssa_detections']:
130+
131+
output_dict[contentType] = []
132+
t:list[SecurityContentObject] = getattr(director,contentType)
133+
134+
for item in t:
135+
output_dict[contentType].append(item.model_dump())
136+
return output_dict
137+

contentctl/contentctl.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,11 @@ def deploy_acs_func(config:deploy_acs):
9999
raise Exception("deploy acs not yet implemented")
100100

101101
def test_common_func(config:test_common):
102+
if type(config) == test:
103+
#construct the container Infrastructure objects
104+
config.getContainerInfrastructureObjects()
105+
#otherwise, they have already been passed as servers
106+
102107
director_output_dto = build_func(config)
103108
gitServer = GitService(director=director_output_dto,config=config)
104109
detections_to_test = gitServer.getContent()
@@ -206,10 +211,6 @@ def main():
206211
updated_config = deploy_acs.model_validate(config)
207212
deploy_acs_func(updated_config)
208213
elif type(config) == test or type(config) == test_servers:
209-
if type(config) == test:
210-
#construct the container Infrastructure objects
211-
config.getContainerInfrastructureObjects()
212-
#otherwise, they have already been passed as servers
213214
test_common_func(config)
214215
else:
215216
raise Exception(f"Unknown command line type '{type(config).__name__}'")

0 commit comments

Comments
 (0)