diff --git a/shared/AppInsightsCore/Tests/Unit/src/ai/AppInsightsCommon.tests.ts b/shared/AppInsightsCore/Tests/Unit/src/ai/AppInsightsCommon.tests.ts index 19a432098..a62b775f5 100644 --- a/shared/AppInsightsCore/Tests/Unit/src/ai/AppInsightsCommon.tests.ts +++ b/shared/AppInsightsCore/Tests/Unit/src/ai/AppInsightsCommon.tests.ts @@ -3,7 +3,7 @@ import { Assert, AITestClass } from "@microsoft/ai-test-framework"; import { DiagnosticLogger } from "../../../../src/diagnostics/DiagnosticLogger"; import { IConfiguration } from "../../../../src/interfaces/ai/IConfiguration"; import { dataSanitizeInput, dataSanitizeKey, dataSanitizeMessage, DataSanitizerValues, dataSanitizeString, dataSanitizeUrl } from "../../../../src/telemetry/ai/Common/DataSanitizer"; - +import { UrlRedactionOptions } from "../../../../src/enums/ai/UrlRedactionOptions" export class ApplicationInsightsTests extends AITestClass { logger = new DiagnosticLogger(); @@ -395,6 +395,7 @@ export class ApplicationInsightsTests extends AITestClass { test: () => { // URLs with sensitive query parameters let config = { + redactUrls: UrlRedactionOptions.appendToDefault, redactQueryParams: ["authorize", "api_key", "password"] } as IConfiguration; const urlWithSensitiveParams = "https://example.com/api?Signature=secret&authorize=value"; @@ -405,5 +406,22 @@ export class ApplicationInsightsTests extends AITestClass { Assert.equal(expectedRedactedUrl, result); } }); + + this.testCase({ + name: 'DataSanitizerTests: dataSanitizeUrl properly redacts sensitive query parameters (only custom)', + test: () => { + // URLs with sensitive query parameters + let config = { + redactUrls: UrlRedactionOptions.replaceDefault, + redactQueryParams: ["authorize", "api_key", "password"] + } as IConfiguration; + const urlWithSensitiveParams = "https://example.com/api?Signature=secret&authorize=value"; + const expectedRedactedUrl = "https://example.com/api?Signature=secret&authorize=REDACTED"; + + // Act & Assert + const result = dataSanitizeUrl(this.logger, urlWithSensitiveParams, config); + Assert.equal(expectedRedactedUrl, result); + } + }); } } diff --git a/shared/AppInsightsCore/Tests/Unit/src/ai/ApplicationInsightsCore.Tests.ts b/shared/AppInsightsCore/Tests/Unit/src/ai/ApplicationInsightsCore.Tests.ts index 15826fc36..ea9fb9710 100644 --- a/shared/AppInsightsCore/Tests/Unit/src/ai/ApplicationInsightsCore.Tests.ts +++ b/shared/AppInsightsCore/Tests/Unit/src/ai/ApplicationInsightsCore.Tests.ts @@ -11,6 +11,7 @@ import { _InternalLogMessage, DiagnosticLogger } from "../../../../src/diagnosti import { ActiveStatus } from "../../../../src/enums/ai/InitActiveStatusEnum"; import { createAsyncPromise, createAsyncRejectedPromise, createAsyncResolvedPromise, createTimeoutPromise, doAwaitResponse } from "@nevware21/ts-async"; import { setBypassLazyCache } from "@nevware21/ts-utils"; +import { UrlRedactionOptions } from "../../../../src/enums/ai/UrlRedactionOptions" const AIInternalMessagePrefix = "AITR_"; const MaxInt32 = 0xFFFFFFFF; @@ -2067,6 +2068,20 @@ export class ApplicationInsightsCoreTests extends AITestClass { "Complex URL should have credentials and sensitive query parameters redacted while preserving other components"); } }); + + this.testCase({ + name: "FieldRedaction: should not redact URLs when redaction is disabled in config, even if they contain credentials and sensitive query parameters", + test: () => { + let config = { + redactUrls: false, + } as IConfiguration; + const url = "https://username:password@example.com:8443/path/to/resource?sig=secret&color=blue#section2"; + const redactedLocation = fieldRedaction(url, config); + Assert.equal(redactedLocation, "https://username:password@example.com:8443/path/to/resource?sig=secret&color=blue#section2", + "URL should not redact credentials and sensitive query parameters when redaction is disabled in config"); + } + }); + this.testCase({ name: "FieldRedaction: should handle completely empty URL string", test: () => { @@ -2197,9 +2212,10 @@ export class ApplicationInsightsCoreTests extends AITestClass { }); this.testCase({ - name: "FieldRedaction: should redact custom query parameters defined in redactQueryParams", + name: "FieldRedaction: should redact custom query parameters defined in redactQueryParams and replace custom queryParams", test: () => { let config = { + redactUrls: UrlRedactionOptions.replaceDefault, redactQueryParams: ["authorize", "api_key", "password"] } as IConfiguration; @@ -2213,6 +2229,7 @@ export class ApplicationInsightsCoreTests extends AITestClass { name: "FieldRedaction: should redact both default and custom query parameters", test: () => { let config = { + redactUrls: UrlRedactionOptions.appendToDefault, redactQueryParams: ["auth_token"] } as IConfiguration; @@ -2223,26 +2240,24 @@ export class ApplicationInsightsCoreTests extends AITestClass { } }); this.testCase({ - name: "FieldRedaction:should not redact custom parameters when redaction is disabled", + name: "FieldRedaction:should replace custom parameters redactQueryParams when user specifies the replace config", test: () => { let config = { - redactUrls: false, + redactUrls: UrlRedactionOptions.replaceDefault, redactQueryParams: ["authorize", "api_key"] } as IConfiguration; - const url = "https://example.com/path?auth_token=12345&authorize=secret"; + const url = "https://username:password@example.com/path?auth_token=12345&authorize=secret"; const redactedLocation = fieldRedaction(url, config); - Assert.equal(redactedLocation, "https://example.com/path?auth_token=12345&authorize=secret", - "URL with custom sensitive parameters should not be redacted when redaction is disabled"); + Assert.equal(redactedLocation, "https://REDACTED:REDACTED@example.com/path?auth_token=12345&authorize=REDACTED", + "URL with custom sensitive parameters should be redacted when query redaction is not disabled"); } }); this.testCase({ name: "FieldRedaction: should handle empty redactQueryParams array", test: () => { - let config = { - redactQueryParams: [] - } as IConfiguration; + let config = {} as IConfiguration; // Should still redact default parameters const url = "https://example.com/path?Signature=secret&custom_param=value"; @@ -2256,6 +2271,7 @@ export class ApplicationInsightsCoreTests extends AITestClass { name: "FieldRedaction:should handle complex URLs with both credentials and custom query parameters", test: () => { let config = { + redactUrls: UrlRedactionOptions.appendToDefault, redactQueryParams: ["authorize", "session_id"] } as IConfiguration; @@ -2584,6 +2600,34 @@ export class ApplicationInsightsCoreTests extends AITestClass { } }); + this.testCase({ + name: "FieldRedaction: should redact credentials while preserving query strings when redactQueryParams is false", + test: () => { + let config = { + redactUrls: 5 + } as IConfiguration; + const url = "https://user:password@example.com/path?sig=secret&color=blue&token=abc123"; + const redactedLocation = fieldRedaction(url, config); + Assert.equal(redactedLocation, "https://REDACTED:REDACTED@example.com/path?sig=secret&color=blue&token=abc123", + "Credentials should be redacted while query string values remain unchanged when redactQueryParams is false"); + } + }); + + this.testCase({ + name: "FieldRedaction: should handle custom parameters with multiple occurrences and empty values", + test: () => { + let config = { + redactUrls: UrlRedactionOptions.replaceDefault, + redactQueryParams: ["auth_token", "session_id"] + } as IConfiguration; + const url = "https://example.com/path?auth_token=first&name=test&auth_token=&session_id=abc&session_id="; + const redactedLocation = fieldRedaction(url, config); + // Only redact parameters that have actual values, not empty ones + Assert.equal(redactedLocation, "https://example.com/path?auth_token=REDACTED&name=test&auth_token=&session_id=REDACTED&session_id=", + "Only non-empty custom sensitive parameters should be redacted"); + } + }); + this.testCase({ name: "FieldRedaction: should handle parameters without values mixed with valued parameters", test: () => { @@ -2598,16 +2642,28 @@ export class ApplicationInsightsCoreTests extends AITestClass { }); this.testCase({ - name: "FieldRedaction: should handle custom parameters with multiple occurrences and empty values", + name: "FieldRedaction: should redact all parts of the URL (username, password, default query params) when redactUrls is set to True", test: () => { let config = { - redactQueryParams: ["auth_token", "session_id"] + redactUrls: true } as IConfiguration; - const url = "https://example.com/path?auth_token=first&name=test&auth_token=&session_id=abc&session_id="; + const url = "https://user:password@example.com/path?sig=secret&color=blue&token=abc123"; const redactedLocation = fieldRedaction(url, config); - // Only redact parameters that have actual values, not empty ones - Assert.equal(redactedLocation, "https://example.com/path?auth_token=REDACTED&name=test&auth_token=&session_id=REDACTED&session_id=", - "Only non-empty custom sensitive parameters should be redacted"); + Assert.equal(redactedLocation, "https://REDACTED:REDACTED@example.com/path?sig=REDACTED&color=blue&token=abc123", + "All parts of the URL should be redacted when redactUrls is true"); + } + }); + + this.testCase({ + name: "FieldRedaction: should not redact credentials or query strings when redactUrls and redactQueryParams are false", + test: () => { + let config = { + redactUrls: UrlRedactionOptions.false + } as IConfiguration; + const url = "https://user:password@example.com/path?sig=secret&color=blue&token=abc123"; + const redactedLocation = fieldRedaction(url, config); + Assert.equal(redactedLocation, url, + "Nothing should be redacted when both redactUrls and redactQueryParams are false"); } }); diff --git a/shared/AppInsightsCore/src/enums/ai/UrlRedactionOptions.ts b/shared/AppInsightsCore/src/enums/ai/UrlRedactionOptions.ts new file mode 100644 index 000000000..0e8d90858 --- /dev/null +++ b/shared/AppInsightsCore/src/enums/ai/UrlRedactionOptions.ts @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Controls how the user can configure which parts of the URL should be redacted. Example, certain query parameters, username and password, etc. +*/ + +export const enum UrlRedactionOptions { + /** + * The default value, will redact the username and password as well as the default set of query parameters + */ + true = 1, + + /** + * Does not redact username and password or any query parameters, the URL will be left as is. Note: this is not recommended as it may lead + * to sensitive data being sent in clear text. + */ + false = 2, + + /** + * This will append any additional queryParams that the user has provided through redactQueryParams config to the default set i.e to + * @defaultValue ["sig", "Signature", "AWSAccessKeyId", "X-Goog-Signature"]. + */ + appendToDefault = 3, + + /** + * This will replace the default set of query parameters to redact with the query parameters defined in redactQueryParams config, if provided by the user. + */ + replaceDefault = 4, + + /** + * This will redact username and password in the URL but will not redact any query parameters, even those in the default set. + */ + usernamePasswordOnly = 5, + + /** + * This will only redact the query parameter in the default set of query parameters to redact. It will not redact username and password. + */ + queryParamsOnly = 6, + +} \ No newline at end of file diff --git a/shared/AppInsightsCore/src/interfaces/ai/IConfiguration.ts b/shared/AppInsightsCore/src/interfaces/ai/IConfiguration.ts index c3db75b74..d4d6c7487 100644 --- a/shared/AppInsightsCore/src/interfaces/ai/IConfiguration.ts +++ b/shared/AppInsightsCore/src/interfaces/ai/IConfiguration.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import { IPromise } from "@nevware21/ts-async"; import { eTraceHeadersMode } from "../../enums/ai/TraceHeadersMode"; +import { UrlRedactionOptions } from "../../enums/ai/UrlRedactionOptions"; import { IOTelConfig } from "../otel/config/IOTelConfig"; import { IAppInsightsCore } from "./IAppInsightsCore"; import { IChannelControls } from "./IChannelControls"; @@ -232,10 +233,10 @@ export interface IConfiguration extends IOTelConfig { expCfg?: IExceptionConfig; /** - * [Optional] A flag to enable or disable the use of the field redaction for urls. + * [Optional] A flag to enable or disable redaction for query parameters. * @defaultValue true */ - redactUrls?: boolean; + redactUrls?: boolean | UrlRedactionOptions; /** * [Optional] Additional query parameters to redact beyond the default set. diff --git a/shared/AppInsightsCore/src/utils/EnvUtils.ts b/shared/AppInsightsCore/src/utils/EnvUtils.ts index ed2382901..5a9cb3e53 100644 --- a/shared/AppInsightsCore/src/utils/EnvUtils.ts +++ b/shared/AppInsightsCore/src/utils/EnvUtils.ts @@ -8,6 +8,7 @@ import { isFunction, isNullOrUndefined, isString, isUndefined, mathMax, strIndexOf, strSubstring } from "@nevware21/ts-utils"; import { DEFAULT_SENSITIVE_PARAMS, STR_EMPTY, STR_REDACTED } from "../constants/InternalConstants"; +import { UrlRedactionOptions } from "../enums/ai/UrlRedactionOptions"; import { IConfiguration } from "../interfaces/ai/IConfiguration"; import { strContains } from "./HelperFuncs"; @@ -455,8 +456,12 @@ function redactQueryParameters(url: string, config?: IConfiguration): string { return url; } - if (config && config.redactQueryParams) { + const option = config ? config.redactUrls : undefined; + + if (option === UrlRedactionOptions.appendToDefault) { sensitiveParams = DEFAULT_SENSITIVE_PARAMS.concat(config.redactQueryParams); + } else if (option === UrlRedactionOptions.replaceDefault) { + sensitiveParams = config.redactQueryParams; } else { sensitiveParams = DEFAULT_SENSITIVE_PARAMS; } @@ -543,17 +548,27 @@ export function fieldRedaction(input: string, config: IConfiguration): string { if (!input || !isString(input) || strIndexOf(input, " ") !== -1) { return input; } - const isRedactionDisabled = config && config.redactUrls === false; + const isRedactionDisabled = config && (config.redactUrls === false || config.redactUrls === UrlRedactionOptions.false); if (isRedactionDisabled) { return input; } - const hasCredentials = strIndexOf(input, "@") !== -1; - const hasQueryParams = strIndexOf(input, "?") !== -1; + + let hasCredentials = strIndexOf(input, "@") !== -1; + let hasQueryParams = strIndexOf(input, "?") !== -1; // If no credentials and no query params, return original if (!hasCredentials && !hasQueryParams) { return input; } + + if (config.redactUrls === UrlRedactionOptions.usernamePasswordOnly) { + hasQueryParams = false; + } + + if (config.redactUrls === UrlRedactionOptions.queryParamsOnly) { + hasCredentials = false; + } + try { let result = input; if (hasCredentials) {