Skip to content

Commit 8e8417f

Browse files
jeffcumpsty-tpxdependabot[bot]
jeffcumpsty-tpx
and
dependabot[bot]
authored
Feature/report additional fields (#140)
* Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Staging (#125) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Sanitize URLs in logging for schema loading and error handling (#124) * Update staging-to-main workflow to allow dependabot merges and modify .gitignore to include log files (#132) * Update .gitignore to ignore all log files in the logs directory (#134) * ci: bump the github-actions-updates group with 2 updates (#126) Bumps the github-actions-updates group with 2 updates: [github/codeql-action](https://github.com/github/codeql-action) and [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action). Updates `github/codeql-action` from 4.32.2 to 4.32.3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](github/codeql-action@v4.32.2...v4.32.3) Updates `aquasecurity/trivy-action` from 0.33.1 to 0.34.0 - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](aquasecurity/trivy-action@0.33.1...0.34.0) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.32.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions-updates - dependency-name: aquasecurity/trivy-action dependency-version: 0.34.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions-updates ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Staging (#137) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Sanitize URLs in logging for schema loading and error handling (#124) * Update staging-to-main workflow to allow dependabot merges and modify .gitignore to include log files (#132) * Update .gitignore to ignore all log files in the logs directory (#134) * Jeffcumpsty tpx patch 1 (#135) * Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Staging (#125) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Sanitize URLs in logging for schema loading and error handling (#124) * Update staging-to-main workflow to allow dependabot merges and modify .gitignore to include log files (#132) * Update .gitignore to ignore all log files in the logs directory (#134) * Enhance error message for non-staging merges Added echo statement to display current actor during merge error. * chore: Bump Swashbuckle.AspNetCore from 10.1.2 to 10.1.3 (#131) * Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * chore: Bump Swashbuckle.AspNetCore from 10.1.2 to 10.1.3 --- updated-dependencies: - dependency-name: Swashbuckle.AspNetCore dependency-version: 10.1.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * Staging (#125) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Sanitize URLs in logging for schema loading and error handling (#124) * Update staging-to-main workflow to allow dependabot merges and modify .gitignore to include log files (#132) * Update .gitignore to ignore all log files in the logs directory (#134) --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Jeff Cumpsty <jeff.cumpsty@tpximpact.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: Bump Serilog.Sinks.File from 6.0.0 to 7.0.0 (#130) * Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * chore: Bump Serilog.Sinks.File from 6.0.0 to 7.0.0 --- updated-dependencies: - dependency-name: Serilog.Sinks.File dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Jeff Cumpsty <jeff.cumpsty@tpximpact.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: Bump Serilog.AspNetCore and Serilog.Sinks.File (#129) * Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * chore: Bump Serilog.AspNetCore and Serilog.Sinks.File Bumps Serilog.AspNetCore from 8.0.3 to 10.0.0 Bumps Serilog.Sinks.File from 6.0.0 to 7.0.0 --- updated-dependencies: - dependency-name: Serilog.AspNetCore dependency-version: 10.0.0 dependency-type: direct:production update-type: version-update:semver-major - dependency-name: Serilog.Sinks.File dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Jeff Cumpsty <jeff.cumpsty@tpximpact.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore: Bump the testing-packages group with 1 update (#128) * Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * chore: Bump the testing-packages group with 1 update Bumps coverlet.collector from 6.0.4 to 8.0.0 --- updated-dependencies: - dependency-name: coverlet.collector dependency-version: 8.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: testing-packages ... Signed-off-by: dependabot[bot] <support@github.com> * Staging (#125) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Sanitize URLs in logging for schema loading and error handling (#124) * Update staging-to-main workflow to allow dependabot merges and modify .gitignore to include log files (#132) * Update .gitignore to ignore all log files in the logs directory (#134) --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Jeff Cumpsty <jeff.cumpsty@tpximpact.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Update .gitignore to ignore all log files in the logs directory (#136) * chore: Bump the microsoft-packages group with 1 update (#127) * Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * chore: Bump the microsoft-packages group with 1 update Bumps System.IdentityModel.Tokens.Jwt from 8.15.0 to 8.16.0 --- updated-dependencies: - dependency-name: System.IdentityModel.Tokens.Jwt dependency-version: 8.16.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: microsoft-packages ... Signed-off-by: dependabot[bot] <support@github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Jeff Cumpsty <jeff.cumpsty@tpximpact.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Merge maion to staging (#139) * Staging (#118) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Staging (#125) * Update permissions in deploy.yml to allow writing for GitHub releases * Add job name to staging-to-main workflow for clarity * Sanitize URLs in logging for schema loading and error handling (#124) * Update staging-to-main workflow to allow dependabot merges and modify .gitignore to include log files (#132) * Update .gitignore to ignore all log files in the logs directory (#134) * ci: bump the github-actions-updates group with 2 updates (#126) Bumps the github-actions-updates group with 2 updates: [github/codeql-action](https://github.com/github/codeql-action) and [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action). Updates `github/codeql-action` from 4.32.2 to 4.32.3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](github/codeql-action@v4.32.2...v4.32.3) Updates `aquasecurity/trivy-action` from 0.33.1 to 0.34.0 - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](aquasecurity/trivy-action@0.33.1...0.34.0) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.32.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions-updates - dependency-name: aquasecurity/trivy-action dependency-version: 0.34.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions-updates ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat: add support for reporting additional fields in JSON validation * feat: add option to report additional fields in OpenAPI validation * feat: flatten resolved allOf properties to enhance composite field discovery --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
1 parent bd2f247 commit 8e8417f

File tree

6 files changed

+387
-8
lines changed

6 files changed

+387
-8
lines changed

‎OpenReferralApi.Core/Models/OpenApiModels.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,16 @@ public class OpenApiValidationOptions
207207
/// </summary>
208208
[JsonProperty("returnRawResult")]
209209
public bool ReturnRawResult { get; set; } = false;
210+
211+
/// <summary>
212+
/// Whether to report fields in endpoint responses that are not defined in the schema.
213+
/// When true, validates response data against the schema and reports any fields that exist in the response
214+
/// but are not defined in the schema as informational messages.
215+
/// When false (default), only standard schema validation is performed.
216+
/// Useful for identifying undocumented fields, schema drift, or incomplete specifications.
217+
/// </summary>
218+
[JsonProperty("reportAdditionalFields")]
219+
public bool ReportAdditionalFields { get; set; } = false;
210220
}
211221

212222
/// <summary>

‎OpenReferralApi.Core/Models/ValidationRequest.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,5 +67,8 @@ public class ValidationOptions
6767

6868
[JsonProperty("validateSslCertificate")]
6969
public bool ValidateSslCertificate { get; set; } = true;
70+
[JsonProperty("reportAdditionalFields")]
71+
public bool ReportAdditionalFields { get; set; } = false;
72+
7073
}
7174

‎OpenReferralApi.Core/Services/JsonValidatorService.cs‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,15 @@ private async Task<ValidationResult> ValidateCoreAsync(ValidationRequest request
9595
var jsonDataString = JsonConvert.SerializeObject(jsonData, Formatting.Indented);
9696
var validationErrors = await ValidateJsonAgainstSchemaAsync(jsonDataString, schema, request.Options);
9797

98-
// Build result
99-
result.IsValid = !validationErrors.Any();
98+
// Report additional fields if requested
99+
if (request.Options?.ReportAdditionalFields == true)
100+
{
101+
var additionalFieldWarnings = DetectAdditionalFields(jsonDataString, schema);
102+
validationErrors.AddRange(additionalFieldWarnings);
103+
}
104+
105+
// Build result - only count Error severity as validation failures
106+
result.IsValid = !validationErrors.Any(e => e.Severity == "Error");
100107
result.Errors = validationErrors;
101108
result.SchemaVersion = "2020-12";
102109
result.Metadata = new ValidationMetadata
@@ -433,5 +440,83 @@ private async Task<object> FetchJsonDataFromUrlAsync(string dataUrl, ValidationO
433440
}
434441
}
435442

443+
/// <summary>
444+
/// Detects fields in the JSON data that are not defined in the schema.
445+
/// Returns a list of validation warnings for each additional field found.
446+
/// </summary>
447+
private List<ValidationError> DetectAdditionalFields(string jsonData, JSchema schema)
448+
{
449+
var warnings = new List<ValidationError>();
450+
451+
try
452+
{
453+
var jsonToken = JToken.Parse(jsonData);
454+
DetectAdditionalFieldsRecursive(jsonToken, schema, "", warnings);
455+
}
456+
catch (Exception ex)
457+
{
458+
_logger.LogWarning(ex, "Error detecting additional fields");
459+
}
460+
461+
return warnings;
462+
}
463+
464+
/// <summary>
465+
/// Recursively traverses the JSON data and schema to detect fields not defined in the schema.
466+
/// </summary>
467+
private void DetectAdditionalFieldsRecursive(JToken jsonToken, JSchema schema, string currentPath, List<ValidationError> warnings)
468+
{
469+
// Handle objects
470+
if (jsonToken.Type == JTokenType.Object && jsonToken is JObject jObject)
471+
{
472+
// Get the properties defined in the schema
473+
var schemaProperties = schema.Properties ?? new Dictionary<string, JSchema>();
474+
var additionalPropertiesAllowed = schema.AllowAdditionalProperties;
475+
var additionalPropertiesSchema = schema.AdditionalProperties;
476+
477+
foreach (var property in jObject.Properties())
478+
{
479+
var propertyPath = string.IsNullOrEmpty(currentPath) ? property.Name : $"{currentPath}.{property.Name}";
480+
481+
// Check if this property is defined in the schema
482+
if (!schemaProperties.ContainsKey(property.Name))
483+
{
484+
// Property not defined in schema - report it
485+
warnings.Add(new ValidationError
486+
{
487+
Path = propertyPath,
488+
Message = $"Field '{property.Name}' is not defined in the schema",
489+
ErrorCode = "ADDITIONAL_FIELD",
490+
Severity = "Info"
491+
});
492+
}
493+
494+
// Recursively check nested properties if there's a schema definition
495+
if (schemaProperties.TryGetValue(property.Name, out var propertySchema))
496+
{
497+
DetectAdditionalFieldsRecursive(property.Value, propertySchema, propertyPath, warnings);
498+
}
499+
else if (additionalPropertiesSchema != null)
500+
{
501+
// If there's an additionalProperties schema, use it for validation
502+
DetectAdditionalFieldsRecursive(property.Value, additionalPropertiesSchema, propertyPath, warnings);
503+
}
504+
}
505+
}
506+
// Handle arrays
507+
else if (jsonToken.Type == JTokenType.Array && jsonToken is JArray jArray)
508+
{
509+
var itemsSchema = schema.Items?.FirstOrDefault();
510+
if (itemsSchema != null)
511+
{
512+
for (int i = 0; i < jArray.Count; i++)
513+
{
514+
var itemPath = $"{currentPath}[{i}]";
515+
DetectAdditionalFieldsRecursive(jArray[i], itemsSchema, itemPath, warnings);
516+
}
517+
}
518+
}
519+
}
520+
436521
}
437522

‎OpenReferralApi.Core/Services/OpenApiValidationService.cs‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -576,7 +576,7 @@ private async Task<EndpointTestResult> TestSingleEndpointAsync(string path, stri
576576
// Validate response if schema is defined
577577
if (testResult.IsSuccess && testResult.ResponseBody != null)
578578
{
579-
await ValidateResponseAsync(testResult, operation, openApiDocument, documentUri, cancellationToken);
579+
await ValidateResponseAsync(testResult, operation, openApiDocument, documentUri, options, cancellationToken);
580580
result.Status = testResult.ValidationResult != null && testResult.ValidationResult.IsValid ? "Success" : "Failed";
581581
}
582582

@@ -685,7 +685,7 @@ private async Task TestPaginatedEndpointAsync(
685685
// Validate first page response schema
686686
if (firstPageResult.ResponseBody != null)
687687
{
688-
await ValidateResponseAsync(firstPageResult, operation, openApiDocument, documentUri, cancellationToken);
688+
await ValidateResponseAsync(firstPageResult, operation, openApiDocument, documentUri, options, cancellationToken);
689689
}
690690

691691
// Try to determine total pages and check for empty feed
@@ -722,7 +722,7 @@ private async Task TestPaginatedEndpointAsync(
722722

723723
if (middlePageResult.IsSuccess && middlePageResult.ResponseBody != null)
724724
{
725-
await ValidateResponseAsync(middlePageResult, operation, openApiDocument, documentUri, cancellationToken);
725+
await ValidateResponseAsync(middlePageResult, operation, openApiDocument, documentUri, options, cancellationToken);
726726
}
727727
}
728728

@@ -734,7 +734,7 @@ private async Task TestPaginatedEndpointAsync(
734734

735735
if (lastPageResult.IsSuccess && lastPageResult.ResponseBody != null)
736736
{
737-
await ValidateResponseAsync(lastPageResult, operation, openApiDocument, documentUri, cancellationToken);
737+
await ValidateResponseAsync(lastPageResult, operation, openApiDocument, documentUri, options, cancellationToken);
738738
}
739739
}
740740
else
@@ -964,7 +964,7 @@ private async Task<HttpTestResult> ExecuteHttpRequestAsync(string url, string me
964964
return testResult;
965965
}
966966

967-
private async Task ValidateResponseAsync(HttpTestResult testResult, JObject operation, JObject openApiDocument, string? documentUri, CancellationToken cancellationToken)
967+
private async Task ValidateResponseAsync(HttpTestResult testResult, JObject operation, JObject openApiDocument, string? documentUri, OpenApiValidationOptions options, CancellationToken cancellationToken)
968968
{
969969
try
970970
{
@@ -996,7 +996,11 @@ private async Task ValidateResponseAsync(HttpTestResult testResult, JObject oper
996996
var validationRequest = new ValidationRequest
997997
{
998998
JsonData = JsonConvert.DeserializeObject(testResult.ResponseBody ?? "{}"),
999-
Schema = schema
999+
Schema = schema,
1000+
Options = new ValidationOptions
1001+
{
1002+
ReportAdditionalFields = options?.ReportAdditionalFields ?? false
1003+
}
10001004
};
10011005
var validationResult = await _jsonValidatorService.ValidateAsync(validationRequest, cancellationToken);
10021006
testResult.ValidationResult = validationResult;

‎OpenReferralApi.Core/Services/SchemaResolverService.cs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -608,12 +608,92 @@ private bool HasSelfReference(JsonNode? schema, string refPointer)
608608
{
609609
result[kvp.Key] = await ResolveAllRefsAsync(kvp.Value, visitedRefs);
610610
}
611+
612+
// Flatten resolved allOf properties to make composite fields discoverable.
613+
MergeAllOfIntoObject(result);
614+
611615
return result;
612616
}
613617

614618
return obj;
615619
}
616620

621+
private static void MergeAllOfIntoObject(JsonObject target)
622+
{
623+
if (!target.TryGetPropertyValue("allOf", out var allOfNode) || allOfNode is not JsonArray allOfArray)
624+
{
625+
return;
626+
}
627+
628+
JsonObject? targetProperties = null;
629+
if (target.TryGetPropertyValue("properties", out var propsNode) && propsNode is JsonObject propsObject)
630+
{
631+
targetProperties = propsObject;
632+
}
633+
634+
JsonArray? targetRequired = null;
635+
if (target.TryGetPropertyValue("required", out var requiredNode) && requiredNode is JsonArray requiredArray)
636+
{
637+
targetRequired = requiredArray;
638+
}
639+
640+
foreach (var item in allOfArray)
641+
{
642+
if (item is not JsonObject itemObject)
643+
{
644+
continue;
645+
}
646+
647+
if (itemObject.TryGetPropertyValue("properties", out var itemPropsNode) && itemPropsNode is JsonObject itemProps)
648+
{
649+
targetProperties ??= new JsonObject();
650+
651+
foreach (var kvp in itemProps)
652+
{
653+
if (!targetProperties.ContainsKey(kvp.Key))
654+
{
655+
targetProperties[kvp.Key] = kvp.Value?.DeepClone();
656+
}
657+
}
658+
}
659+
660+
if (itemObject.TryGetPropertyValue("required", out var itemRequiredNode) && itemRequiredNode is JsonArray itemRequired)
661+
{
662+
targetRequired ??= new JsonArray();
663+
664+
foreach (var requiredItem in itemRequired)
665+
{
666+
if (requiredItem is not JsonValue requiredValue)
667+
{
668+
continue;
669+
}
670+
671+
var requiredName = requiredValue.GetValue<string>();
672+
if (!targetRequired.Any(existing => existing?.GetValue<string>() == requiredName))
673+
{
674+
targetRequired.Add(requiredName);
675+
}
676+
}
677+
}
678+
679+
if (!target.TryGetPropertyValue("type", out _) &&
680+
itemObject.TryGetPropertyValue("type", out var itemTypeNode))
681+
{
682+
target["type"] = itemTypeNode?.DeepClone();
683+
}
684+
}
685+
686+
if (targetProperties != null)
687+
{
688+
target["properties"] = targetProperties;
689+
}
690+
691+
if (targetRequired != null)
692+
{
693+
target["required"] = targetRequired;
694+
}
695+
}
696+
617697
/// <summary>
618698
/// Creates a JSON schema from JSON string with proper reference resolution
619699
/// </summary>

0 commit comments

Comments
 (0)