Merge pull request #104 from mitre/asff_mapper

Author: rx294

.github/workflows/build.yml

@@ -86,3 +86,15 @@ jobs:
           jq 'del(.version, .platform.release)' scoutsuite_output.json > scoutsuite_output_jq.json
           jq 'del(.version, .platform.release)' ./sample_jsons/scoutsuite_mapper/scoutsuite_hdf.json > scoutsuite_sample.json
           diff scoutsuite_sample.json scoutsuite_output_jq.json
+      - name: Test asff mapper
+        run: |
+          heimdall_tools asff_mapper -i ./sample_jsons/asff_mapper/sample_input_jsons/asff_sample.json --sh ./sample_jsons/asff_mapper/sample_input_jsons/aws_cis_standard.json -o asff_output.json
+          jq 'del(.version, .platform.release)' asff_output.json > asff_output_jq.json
+          jq 'del(.version, .platform.release)' ./sample_jsons/asff_mapper/asff_hdf.json > asff_sample.json
+          diff asff_sample.json asff_output_jq.json
+      - name: Test prowler mapper
+        run: |
+          heimdall_tools prowler_mapper -i ./sample_jsons/prowler_mapper/sample_input_jsons/prowler_sample.asff-json -o prowler_output.json
+          jq 'del(.version, .platform.release)' prowler_output.json > prowler_output_jq.json
+          jq 'del(.version, .platform.release)' ./sample_jsons/prowler_mapper/prowler_hdf.json > prowler_sample.json
+          diff prowler_sample.json prowler_output_jq.json

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2021-06-07 20:33:06 UTC using RuboCop version 1.16.0.
+# on 2021-08-10 22:03:51 UTC using RuboCop version 1.14.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -13,18 +13,13 @@ Gemspec/RequiredRubyVersion:
   Exclude:
     - 'heimdall_tools.gemspec'
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: IgnoreLiteralBranches, IgnoreConstantBranches.
 Lint/DuplicateBranch:
   Exclude:
+    - 'lib/heimdall_tools/asff_mapper.rb'
     - 'lib/heimdall_tools/dbprotect_mapper.rb'
 
-# Offense count: 1
-# Configuration parameters: MaximumRangeSize.
-Lint/MissingCopEnableDirective:
-  Exclude:
-    - 'lib/heimdall_tools/nessus_mapper.rb'
-
 # Offense count: 1
 Lint/RequireParentheses:
   Exclude:
@@ -37,10 +32,10 @@ Lint/UnusedMethodArgument:
   Exclude:
     - 'lib/heimdall_tools/hdf.rb'
 
-# Offense count: 37
+# Offense count: 50
 # Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
 Metrics/AbcSize:
-  Max: 124
+  Max: 165
 
 # Offense count: 5
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
@@ -53,17 +48,17 @@ Metrics/BlockLength:
 Metrics/BlockNesting:
   Max: 5
 
-# Offense count: 9
+# Offense count: 10
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
   Max: 175
 
-# Offense count: 10
+# Offense count: 13
 # Configuration parameters: IgnoredMethods.
 Metrics/CyclomaticComplexity:
-  Max: 17
+  Max: 30
 
-# Offense count: 40
+# Offense count: 45
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
 Metrics/MethodLength:
   Max: 56
@@ -73,10 +68,10 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Max: 18
 
-# Offense count: 8
+# Offense count: 11
 # Configuration parameters: IgnoredMethods.
 Metrics/PerceivedComplexity:
-  Max: 17
+  Max: 30
 
 # Offense count: 3
 Naming/AccessorMethodName:
@@ -98,10 +93,11 @@ Naming/VariableName:
   Exclude:
     - 'lib/heimdall_tools/burpsuite_mapper.rb'
 
-# Offense count: 12
+# Offense count: 15
 # Configuration parameters: AllowedVariables.
 Style/GlobalVars:
   Exclude:
+    - 'lib/heimdall_tools/asff_mapper.rb'
     - 'lib/heimdall_tools/jfrog_xray_mapper.rb'
     - 'lib/heimdall_tools/nessus_mapper.rb'
     - 'lib/heimdall_tools/nikto_mapper.rb'

README.md

@@ -5,20 +5,22 @@
 
 HeimdallTools supplies several methods to convert output from various tools to "Heimdall Data Format"(HDF) format to be viewable in Heimdall. The current converters are:
 
-1.  [**aws_config_mapper**](#aws_config_mapper) - assess, audit, and evaluate AWS resources
-1.  [**burpsuite_mapper**](#burpsuite_mapper) - commercial dynamic analysis tool
-1.  [**dbprotect_mapper**](#dbprotect_mapper) - database vulnerability scanner
-1.  [**fortify_mapper**](#fortify_mapper) - commercial static code analysis tool
-1.  [**jfrog_xray_mapper**](#jfrog_xray_mapper) - package vulnerability scanner
-1.  [**nessus_mapper**](#nessus_mapper) - commercial security scanner (supports compliance and vulnerability scans from Tenable.sc and Tenable.io)
-1.  [**netsparker_mapper**](#netsparker_mapper) - web application security scanner
-1.  [**nikto_mapper**](#nikto_mapper) - open-source web server scanner
-1.  [**sarif_mapper**](#sarif_mapper) - static analysis results interchange format
+1. [**asff_mapper**](#asff_mapper) - custom findings format for AWS Security Hub
+1. [**aws_config_mapper**](#aws_config_mapper) - assess, audit, and evaluate AWS resources
+1. [**burpsuite_mapper**](#burpsuite_mapper) - commercial dynamic analysis tool
+1. [**dbprotect_mapper**](#dbprotect_mapper) - database vulnerability scanner
+1. [**fortify_mapper**](#fortify_mapper) - commercial static code analysis tool
+1. [**jfrog_xray_mapper**](#jfrog_xray_mapper) - package vulnerability scanner
+1. [**nessus_mapper**](#nessus_mapper) - commercial security scanner (supports compliance and vulnerability scans from Tenable.sc and Tenable.io)
+1. [**netsparker_mapper**](#netsparker_mapper) - web application security scanner
+1. [**nikto_mapper**](#nikto_mapper) - open-source web server scanner
+1. [**prowler_mapper**](#prowler_mapper) - assess, audit, harden, and facilitate incidence response for AWS resources
+1. [**sarif_mapper**](#sarif_mapper) - static analysis results interchange format
 1. [**scoutsuite_mapper**](#scoutsuite_mapper) - multi-cloud security auditing tool
 1. [**snyk_mapper**](#snyk_mapper) - commercial package vulnerability scanner
 1. [**sonarqube_mapper**](#sonarqube_mapper) - open-source static code analysis tool
 1. [**xccdf_results_mapper**](#xccdf_results_mapper) - extensible configuration checklist description results format
-1. [*scc_mapper](#xccdf_results_mapper) - scap compliance checker format
+1. [**scc_mapper**](#xccdf_results_mapper) - scap compliance checker format
 1. [**zap_mapper**](#zap_mapper) - OWASP ZAP - open-source dynamic code analysis tool
 
 ## Want to recommend a mapper for another tool? Please use these steps:
@@ -84,6 +86,27 @@ For Docker usage, replace the `heimdall_tools` command with the correct Docker c
 
 Note that all of the above Docker commands will mount your current directory on the Docker container. Ensure that you have navigated to the directory you intend to convert files in before executing the command.
 
+## asff_mapper
+
+asff_mapper translates AWS Security Finding Format results from JSON to HDF-formatted JSON so as to be viewable on Heimdall
+
+Note: The following commands are examples to extract data via the AWS CLI that need to be fed to the mapper:
+
+Output|Use|Command
+---|---|---
+ASFF json|All the findings that will be fed into the mapper|aws securityhub get-findings > asff.json
+AWS SecurityHub enabled standards json|Get all the enabled standards so you can get their identifiers|aws securityhub get-enabled-standards > asff_standards.json
+AWS SecurityHub standard controls json|Get all the controls for a standard that will be fed into the mapper|aws securityhub describe-standards-controls --standards-subscription-arn "arn:aws:securityhub:us-east-1:123456789123:subscription/cis-aws-foundations-benchmark/v/1.2.0" > asff_cis_standard.json
+
+    USAGE: heimdall_tools asff_mapper -i <asff-finding-json> [--sh <standard-1-json> ... <standard-n-json>] -o <hdf-scan-results-json>
+
+    FLAGS:
+        -i --input -j --json <asff-finding-json>                   : path to ASFF findings file.
+        --sh --securityhub-standards --input-securityhub-standards : array of paths to AWS SecurityHub standard files.
+        -o --output <hdf-scan-results-json>                        : path to output scan-results json.
+
+    example: heimdall_tools asff_mapper -i asff_findings.json --sh aws_standard.json cis_standard.json -o asff_hdf.json
+
 ## aws_config_mapper
 
 aws_config_mapper pulls Ruby AWS SDK data to translate AWS Config Rule results into HDF format json to be viewable in Heimdall
@@ -99,8 +122,8 @@ aws_config_mapper pulls Ruby AWS SDK data to translate AWS Config Rule results i
     USAGE: heimdall_tools aws_config_mapper [OPTIONS] -o
 
     FLAGS:
-        -o --output        : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -o --output  : path to output scan-results json.
+        -V --verbose : verbose run [optional].
 
     example: heimdall_tools aws_config_mapper -o aws_config_results_hdf.json
 
@@ -111,9 +134,9 @@ burpsuite_mapper translates an BurpSuite Pro exported XML results file into HDF
     USAGE: heimdall_tools burpsuite_mapper [OPTIONS] -x  -o
 
     FLAGS:
-        -x                : path to BurpSuitePro exported XML results file.
-        -o --output        : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -x           : path to BurpSuitePro exported XML results file.
+        -o --output  : path to output scan-results json.
+        -V --verbose : verbose run [optional].
 
     example: heimdall_tools burpsuite_mapper -x burpsuite_results.xml -o scan_results.json
 
@@ -124,9 +147,9 @@ dbprotect_mapper translates DBProtect report in `Check Results Details` format X
     USAGE: heimdall_tools dbprotect_mapper [OPTIONS] -x  -o
 
     FLAGS:
-        -x            : path to DBProtect report XML file.
-        -o --output        : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -x           : path to DBProtect report XML file.
+        -o --output  : path to output scan-results json.
+        -V --verbose : verbose run [optional].
 
     example: heimdall_tools dbprotect_mapper -x check_results_details_report.xml -o db_protect_hdf.json
 
@@ -137,9 +160,9 @@ fortify_mapper translates an Fortify results FVDL file into HDF format json to b
     USAGE: heimdall_tools fortify_mapper [OPTIONS] -f  -o
 
     FLAGS:
-    	-f --fvdl          : path to Fortify Scan FVDL file.
-    	-o --output        : path to output scan-results json.
-    	-V --verbose                     : verbose run [optional].
+    	-f --fvdl    : path to Fortify Scan FVDL file.
+    	-o --output  : path to output scan-results json.
+    	-V --verbose : verbose run [optional].
 
     example: heimdall_tools fortify_mapper -f audit.fvdl -o scan_results.json
 
@@ -150,9 +173,9 @@ jfrog_xray_mapper translates an JFrog Xray results JSON file into HDF format JSO
     USAGE: heimdall_tools jfrog_xray_mapper [OPTIONS] -j  -o
 
     FLAGS:
-        -j            : path to xray results JSON file.
-        -o --output        : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -j           : path to xray results JSON file.
+        -o --output  : path to output scan-results json.
+        -V --verbose : verbose run [optional].
 
     example: heimdall_tools jfrog_xray_mapper -j xray_results.json -o xray_results_hdf.json
 
@@ -166,9 +189,9 @@ Note: A separate HDF JSON file is generated for each host reported in the Nessus
     USAGE: heimdall_tools nessus_mapper [OPTIONS] -x  -o
 
     FLAGS:
-        -x           : path to Nessus-exported XML results file.
-        -o --output_prefix       : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -x                 : path to Nessus-exported XML results file.
+        -o --output_prefix : path to output scan-results json.
+        -V --verbose       : verbose run [optional].
 
     example: heimdall_tools nessus_mapper -x nessus-results.xml -o test-env
 
@@ -181,9 +204,9 @@ The current iteration only works with Netsparker Enterprise Vulnerabilities Scan
     USAGE: heimdall_tools netsparker_mapper [OPTIONS] -x  -o
 
     FLAGS:
-        -x       : path to netsparker results XML file.
-        -o --output        : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -x           : path to netsparker results XML file.
+        -o --output  : path to output scan-results json.
+        -V --verbose : verbose run [optional].
 
     example: heimdall_tools netsparker_mapper -x netsparker_results.xml -o netsparker_hdf.json
 
@@ -196,22 +219,36 @@ Note: Current this mapper only support single target Nikto Scans.
     USAGE: heimdall_tools nikto_mapper [OPTIONS] -x  -o
 
     FLAGS:
-        -j           : path to Nikto results JSON file.
-        -o --output_prefix       : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -j                 : path to Nikto results JSON file.
+        -o --output_prefix : path to output scan-results json.
+        -V --verbose       : verbose run [optional].
 
     example: heimdall_tools nikto_mapper -j nikto_results.json -o nikto_results.json
 
+## prowler_mapper
+
+prowler_mapper translates Prowler-derived AWS Security Finding Format results from concatenated JSON blobs to HDF-formatted JSON so as to be viewable on Heimdall
+
+Note: Currently this mapper only supports Prowler's ASFF output format.
+
+    USAGE: heimdall_tools prowler_mapper -i <prowler-asff-json> -o <hdf-scan-results-json>
+
+    FLAGS:
+        -i --input -j --json <prowler-asff-json> : path to Prowler ASFF findings file.
+        -o --output <hdf-scan-results-json>      : path to output scan-results json.
+
+    example: heimdall_tools prowler_mapper -i prowler_results.js -o prowler_hdf.json
+
 ## sarif_mapper
 
 sarif_mapper translates a SARIF JSON file into HDF format JSON to be viewable in Heimdall
 
     USAGE: heimdall_tools sarif_mapper [OPTIONS] -j  -o
 
     FLAGS:
-        -j           : path to SARIF results JSON file.
-        -o --output_prefix       : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -j                 : path to SARIF results JSON file.
+        -o --output_prefix : path to output scan-results json.
+        -V --verbose       : verbose run [optional].
 
     example: heimdall_tools sarif_mapper -j sarif_results.json -o sarif_results_hdf.json
 
@@ -224,8 +261,8 @@ Note: Currently this mapper only supports AWS.
     USAGE: heimdall_tools scoutsuite_mapper -i  -o
 
     FLAGS:
-        -i --input -j --javascript  : path to Scout Suite results Javascript file.
-        -o --output                 : path to output scan-results json.
+        -i --input -j --javascript : path to Scout Suite results Javascript file.
+        -o --output                : path to output scan-results json.
 
     example: heimdall_tools scoutsuite_mapper -i scoutsuite_results.js -o scoutsuite_hdf.json
 
@@ -238,9 +275,9 @@ Note: A separate HDF JSON is generated for each project reported in the Snyk Rep
     USAGE: heimdall_tools snyk_mapper [OPTIONS] -x  -o
 
     FLAGS:
-        -j           : path to Snyk results JSON file.
-        -o --output_prefix       : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -j                 : path to Snyk results JSON file.
+        -o --output_prefix : path to output scan-results json.
+        -V --verbose       : verbose run [optional].
 
     example: heimdall_tools snyk_mapper -j snyk_results.json -o output-file-prefix
 
@@ -251,11 +288,11 @@ sonarqube_mapper pulls SonarQube results, for the specified project, from the AP
     USAGE: heimdall_tools sonarqube_mapper [OPTIONS] -n  -u  -o
 
     FLAGS:
-        -n --name          : Project Key of the project in SonarQube
-        -u --api_url            : url of the SonarQube Server API. Typically ends with /api.
-        --auth               : username:password or token [optional].
-        -o --output        : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -n --name    : Project Key of the project in SonarQube
+        -u --api_url : url of the SonarQube Server API. Typically ends with /api.
+        --auth       : username:password or token [optional].
+        -o --output  : path to output scan-results json.
+        -V --verbose : verbose run [optional].
 
     example:
 
@@ -272,8 +309,8 @@ xccdf_results_mapper translates an XCCDF_Results XML scan into HDF format json t
     USAGE: heimdall_tools xccdf_results_mapper [OPTIONS] -x  -o
 
     FLAGS:
-        -x            : path to XCCDF-Results XML file.
-        -o --output        : path to output scan-results json.
+        -x          : path to XCCDF-Results XML file.
+        -o --output : path to output scan-results json.
 
     example: heimdall_tools xccdf_results_mapper -x xccdf_results.xml -o scan_results.json
 
@@ -284,10 +321,10 @@ zap_mapper translates OWASP ZAP results Json to HDF format Json be viewed on Hei
     USAGE: heimdall_tools zap_mapper [OPTIONS] -j  -n  -o
 
     FLAGS:
-        -j --json              : path to OWASP ZAP results JSON file.
-        -n --name             : URL of the site being evaluated.
-        -o --output        : path to output scan-results json.
-        -V --verbose                     : verbose run [optional].
+        -j --json    : path to OWASP ZAP results JSON file.
+        -n --name    : URL of the site being evaluated.
+        -o --output  : path to output scan-results json.
+        -V --verbose : verbose run [optional].
 
     example: heimdall_tools zap_mapper -j zap_results.json -n site_name -o scan_results.json
 
@@ -355,6 +392,7 @@ To release a new version, update the version number in `version.rb` according to
 
 ### Authors
 
+-   Author:: Amndeep Singh Mann [Amndeep7](https://github.com/Amndeep7)
 -   Author:: Rony Xavier [rx294](https://github.com/rx294)
 -   Author:: Dan Mirsky [mirskiy](https://github.com/mirskiy)
 

heimdall_tools.gemspec

@@ -15,9 +15,9 @@ Gem::Specification.new do |spec|
   rescue StandardError
     '0.0.0.1.ENOGVB'
   end
-  spec.authors       = ['Robert Thew', 'Rony Xavier', 'Aaron Lippold']
+  spec.authors       = ['Robert Thew', 'Rony Xavier', 'Amndeep Singh Mann', 'Aaron Lippold']
   spec.email         = ['[email protected]']
-  spec.summary       = 'Convert Forify, Openzap and Sonarqube results to HDF'
+  spec.summary       = 'Convert a variety of security product results to HDF'
   spec.description   = 'Converter utils that can be included as a gem or used from the command line'
   spec.homepage      = 'https://github.com/mitre/heimdall_tools'
   spec.license       = 'Apache-2.0'
@@ -29,8 +29,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'aws-sdk-configservice', '~> 1'
+  spec.add_runtime_dependency 'aws-sdk-securityhub', '~> 1'
   spec.add_runtime_dependency 'csv', '~> 3.1'
   spec.add_runtime_dependency 'git-lite-version-bump', '>= 0.17.2'
+  spec.add_runtime_dependency 'htmlentities', '~> 4.3.4'
   spec.add_runtime_dependency 'httparty', '~> 0.18.0'
   spec.add_runtime_dependency 'json', '~> 2.3'
   spec.add_runtime_dependency 'nokogiri', '~> 1.11'