diff --git a/.github/workflows/daily_ci.yml b/.github/workflows/daily_ci.yml index be189b2f9a..48d96a79c7 100644 --- a/.github/workflows/daily_ci.yml +++ b/.github/workflows/daily_ci.yml @@ -67,3 +67,11 @@ jobs: uses: ./.github/workflows/library_interop_tests.yml with: dafny: ${{needs.getVersion.outputs.version}} + daily-fuzz-interop-test: + needs: getVersion + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/library_interop_tests.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + fuzz-testing: true + num-fuzz-vectors: 10000 diff --git a/.github/workflows/library_interop_tests.yml b/.github/workflows/library_interop_tests.yml index 2c72e1c24e..4c35a03769 100644 --- a/.github/workflows/library_interop_tests.yml +++ b/.github/workflows/library_interop_tests.yml @@ -18,6 +18,11 @@ on: required: false default: false type: boolean + num-fuzz-vectors: + description: "Number of fuzz test vectors to generate (only used when fuzz-testing is true)" + required: false + default: 2000 + type: number jobs: generateEncryptVectors: @@ -37,6 +42,11 @@ jobs: permissions: id-token: write contents: read + env: + # Increase Rust stack size for large-scale fuzz testing with 10,000+ test vectors + RUST_MIN_STACK: 404857600 + # Increase Go stack size for large-scale fuzz testing with 10,000+ test vectors + GOSTACKSIZE: 16m steps: - name: Support longpaths on Git checkout run: | @@ -200,8 +210,8 @@ jobs: working-directory: ./${{ matrix.library }} run: | if [ "${{ inputs.fuzz-testing }}" = "true" ]; then - echo "Generating fuzzed test vectors" - make test_generate_fuzz_vectors_${{ matrix.language }} NUM_VECTORS=2000 + echo "Generating fuzzed test vectors (${{ inputs.num-fuzz-vectors }} vectors)" + make test_generate_fuzz_vectors_${{ matrix.language }} NUM_VECTORS=${{ inputs.num-fuzz-vectors }} else make test_generate_vectors_${{ matrix.language }} fi @@ -235,6 +245,11 @@ jobs: permissions: id-token: write contents: read + env: + # Increase Rust stack size for large-scale fuzz testing with 10,000+ test vectors + RUST_MIN_STACK: 404857600 + # Increase Go stack size for large-scale fuzz testing with 10,000+ test vectors + GOSTACKSIZE: 16m steps: - name: Support longpaths on Git checkout run: | diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 333e9bc7ea..29eecadef6 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -62,6 +62,7 @@ jobs: with: dafny: ${{needs.getVersion.outputs.version}} fuzz-testing: true + num-fuzz-vectors: 10000 secrets: inherit pr-ci-all-required: if: always() diff --git a/TestVectorsAwsCryptographicMaterialProviders/dafny/TestVectorsAwsCryptographicMaterialProviders/test/fuzz_generator.py b/TestVectorsAwsCryptographicMaterialProviders/dafny/TestVectorsAwsCryptographicMaterialProviders/test/fuzz_generator.py index 3cddca797c..68a3926ac0 100644 --- a/TestVectorsAwsCryptographicMaterialProviders/dafny/TestVectorsAwsCryptographicMaterialProviders/test/fuzz_generator.py +++ b/TestVectorsAwsCryptographicMaterialProviders/dafny/TestVectorsAwsCryptographicMaterialProviders/test/fuzz_generator.py @@ -32,7 +32,6 @@ "kms": "KMS keyring test with Unicode fuzzing" } -#TODO-Fuzztesting: #include the other keys: rsa for raw keys. Other test types too # Key, Algorithm, Test-Type, Key-Material Definitions KMS_KEYS = ["us-west-2-mrk", "us-east-1-mrk", "us-west-2-decryptable"] #us-west-2-rsa-mrk (already have rsa), us-west-2-256-ecc, us-west-2-384-ecc (and already have enough ecc) RAW_KEY_TYPES = ["aes-128", "aes-192", "aes-256", "ecc-256", "ecc-384", "ecc-521"] #rsa-4096 not included because of complex interdependencies and structural requirements @@ -97,7 +96,7 @@ # Unicode strategies for maximum diversity unicode_strategies = [ - st.text(min_size=1, max_size=50), # Normal text + st.text(min_size=1, max_size=50), st.text(min_size=1, max_size=50, alphabet=st.characters(categories=['So', 'Sc', 'Sk', 'Sm'])), #Symbols st.text(min_size=1, max_size=50, alphabet=st.characters(categories=['Lo', 'Ll', 'Lu', 'Lm', 'Lt'])), #Letters st.text(min_size=1, max_size=50, alphabet=st.characters(categories=['Nd', 'Nl', 'No'])), #Numbers @@ -143,7 +142,6 @@ def fuzz_key_identifiers(draw, base_key_id: str) -> Dict[str, Any]: return {KEY_ID_FIELD: fuzzed_key_id, KEY_NAMESPACE_FIELD: key_namespace, KEY_ID_IN_MATERIAL_FIELD: key_id_in_material} -#TODO-Fuzztesting: Strengthening encryption context fuzzing with specific edge cases (close to the character limitation for encryption context (8,192)) @composite def fuzz_encryption_context(draw): """Generate diverse encryption contexts with Unicode characters. @@ -162,7 +160,6 @@ def fuzz_encryption_context(draw): return context -#TODO-Fuzztesting: "negative-encrypt-keyring" fuzzing functionality: in fuzzToDos branch, implemented tests with missing required keys (for KMS keyrings) or invalid key material (raw keryings), but it could also fail because of algo mismatches or invalid encryption context formats def generate_required_keys(draw, encryption_context: Dict[str, str]) -> List[str]: """Generate requiredEncryptionContextKeys from existing context keys.""" context_keys = list(encryption_context.keys()) @@ -178,7 +175,6 @@ def create_key_description(draw, keyring_type: str, kms_key: str, required_keys: else: raise ValueError(f"Unknown keyring type: {keyring_type}") -#TODO-Fuzztesting: for both raw and kms keys, different keys have different description structures; this has to be taken into consideration for the remaining keys def create_raw_key_description(draw) -> Dict[str, Any]: """Create raw keyring description.""" raw_key_id = draw(st.sampled_from(RAW_KEY_TYPES)) @@ -325,7 +321,6 @@ def generate_fuzz_test_vectors(num_vectors) -> Tuple[Dict[str, Any], Dict[str, A with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=NonInteractiveExampleWarning) for i in range(num_vectors): - # TODO-Fuzztesting: remove .example() usage. Context: we're using Hypothesis as a data generator, not for testing properties. # Hypothesis is designed for property-based testing, so when using .example() it informs us that we should be using @given to actually test properties, not just generate examples. # But we're in a different use case, because we're essentially using Hypothesis as a sophisticated random data generator to create test vectors that will be evaluated by a different test system # Warning surpressed so that it doesn't fill the output with this message: https://github.com/aws/aws-cryptographic-material-providers-library/pull/1630/files#r2226909207; it will paste that message 2000 times for 2000 test vectors, for e.g. @@ -336,8 +331,6 @@ def generate_fuzz_test_vectors(num_vectors) -> Tuple[Dict[str, Any], Dict[str, A new_keys = extract_new_keys(test_vectors) return test_vectors, new_keys -#TODO-Fuzztesting: increase the number of test vectors (for CI), need to increase the stack perhaps? -#TODO-Fuzztesting: Add a logging mechanism to log errors/vulnerabilities we run into def main(): """Main function to generate fuzzed test vectors.""" # Parse command-line arguments diff --git a/TestVectorsAwsCryptographicMaterialProviders/runtimes/java/build.gradle.kts b/TestVectorsAwsCryptographicMaterialProviders/runtimes/java/build.gradle.kts index a576a03895..7b10c34733 100644 --- a/TestVectorsAwsCryptographicMaterialProviders/runtimes/java/build.gradle.kts +++ b/TestVectorsAwsCryptographicMaterialProviders/runtimes/java/build.gradle.kts @@ -97,6 +97,8 @@ tasks.register("runTests") { dependsOn("copyKeysJSON") mainClass.set("TestsFromDafny") classpath = sourceSets["test"].runtimeClasspath + // Increase stack size for large-scale fuzz testing with 10,000+ test vectors + jvmArgs("-Xss16m") } tasks.register("copyKeysJSON") { @@ -111,4 +113,6 @@ tasks.register("copyKeysJSONCurr") { application { mainClass.set("ImplementationFromDafny") + // Increase stack size for large-scale fuzz testing with 10,000+ test vectors + applicationDefaultJvmArgs = listOf("-Xss16m") } diff --git a/TestVectorsAwsCryptographicMaterialProviders/runtimes/net/TestVectors.csproj b/TestVectorsAwsCryptographicMaterialProviders/runtimes/net/TestVectors.csproj index c48826be94..4e5d4d6a5b 100644 --- a/TestVectorsAwsCryptographicMaterialProviders/runtimes/net/TestVectors.csproj +++ b/TestVectorsAwsCryptographicMaterialProviders/runtimes/net/TestVectors.csproj @@ -8,6 +8,8 @@ net6.0;net48 false Exe + + 16777216 diff --git a/TestVectorsAwsCryptographicMaterialProviders/runtimes/python/src/aws_cryptography_materialproviders_test_vectors/__init__.py b/TestVectorsAwsCryptographicMaterialProviders/runtimes/python/src/aws_cryptography_materialproviders_test_vectors/__init__.py index 5d14fb9687..8869ea10c1 100644 --- a/TestVectorsAwsCryptographicMaterialProviders/runtimes/python/src/aws_cryptography_materialproviders_test_vectors/__init__.py +++ b/TestVectorsAwsCryptographicMaterialProviders/runtimes/python/src/aws_cryptography_materialproviders_test_vectors/__init__.py @@ -9,10 +9,8 @@ The MPL Test Vectors generates more than the default recursion of 1000. MPL test vectors exceeds Python's recursion limit when parsing the JSON, which needs >1 call per test vector. (Other Crypto Tools languages are limited by memory; Python's explicit limit on function calls is unique.) -When using this internal Crypto Tools TestVectors library, set recursion limit to 10,000. -(This value is totally arbitrary and should be increased if this isn't enough.) """ -sys.setrecursionlimit(10000) +sys.setrecursionlimit(100000) #previous recursion limit 10,000 # Initialize generated Dafny from .internaldafny.generated import module_