Make URL Redaction more dynamic

shared/AppInsightsCore/Tests/Unit/src/ai/AppInsightsCommon.tests.ts

@@ -3,7 +3,7 @@ import { Assert, AITestClass } from "@microsoft/ai-test-framework";
 import { DiagnosticLogger } from "../../../../src/diagnostics/DiagnosticLogger";
 import { IConfiguration } from "../../../../src/interfaces/ai/IConfiguration";
 import { dataSanitizeInput, dataSanitizeKey, dataSanitizeMessage, DataSanitizerValues, dataSanitizeString, dataSanitizeUrl } from "../../../../src/telemetry/ai/Common/DataSanitizer";
-
+import { UrlRedactionOptions } from "../../../../src/enums/ai/UrlRedactionOptions"
 
 export class ApplicationInsightsTests extends AITestClass {
     logger = new DiagnosticLogger();
@@ -395,6 +395,7 @@ export class ApplicationInsightsTests extends AITestClass {
             test: () => {
                 // URLs with sensitive query parameters
                 let config = {
+                    redactUrls: UrlRedactionOptions.append,
                     redactQueryParams: ["authorize", "api_key", "password"]
                 } as IConfiguration;
                 const urlWithSensitiveParams = "https://example.com/api?Signature=secret&authorize=value";
@@ -405,5 +406,22 @@ export class ApplicationInsightsTests extends AITestClass {
                 Assert.equal(expectedRedactedUrl, result);
             }
         });
+
+        this.testCase({
+            name: 'DataSanitizerTests: dataSanitizeUrl properly redacts sensitive query parameters (only custom)',
+            test: () => {
+                // URLs with sensitive query parameters
+                let config = {
+                    redactUrls: UrlRedactionOptions.replace,
+                    redactQueryParams: ["authorize", "api_key", "password"]
+                } as IConfiguration;
+                const urlWithSensitiveParams = "https://example.com/api?Signature=secret&authorize=value";
+                const expectedRedactedUrl = "https://example.com/api?Signature=secret&authorize=REDACTED";
+
+                // Act & Assert
+                const result = dataSanitizeUrl(this.logger, urlWithSensitiveParams, config);
+                Assert.equal(expectedRedactedUrl, result);
+            }
+        });
     }
 }

shared/AppInsightsCore/Tests/Unit/src/ai/ApplicationInsightsCore.Tests.ts

@@ -11,6 +11,7 @@ import { _InternalLogMessage, DiagnosticLogger } from "../../../../src/diagnosti
 import { ActiveStatus } from "../../../../src/enums/ai/InitActiveStatusEnum";
 import { createAsyncPromise, createAsyncRejectedPromise, createAsyncResolvedPromise, createTimeoutPromise, doAwaitResponse } from "@nevware21/ts-async";
 import { setBypassLazyCache } from "@nevware21/ts-utils";
+import { UrlRedactionOptions } from "../../../../src/enums/ai/UrlRedactionOptions"
 
 const AIInternalMessagePrefix = "AITR_";
 const MaxInt32 = 0xFFFFFFFF;
@@ -2067,6 +2068,20 @@ export class ApplicationInsightsCoreTests extends AITestClass {
                     "Complex URL should have credentials and sensitive query parameters redacted while preserving other components");
             }
         });
+
+         this.testCase({
+            name: "FieldRedaction: should not redact URLs when redaction is disabled in config, even if they contain credentials and sensitive query parameters",
+            test: () => {
+                let config = {
+                    redactUrls: false,
+                } as IConfiguration;
+                const url = "https://username:password@example.com:8443/path/to/resource?sig=secret&color=blue#section2";
+                const redactedLocation = fieldRedaction(url, config);
+                Assert.equal(redactedLocation, "https://username:password@example.com:8443/path/to/resource?sig=secret&color=blue#section2",
+                    "URL should not redact credentials and sensitive query parameters when redaction is disabled in config");
+            }
+        });
+
         this.testCase({
             name: "FieldRedaction: should handle completely empty URL string",
             test: () => {
@@ -2197,9 +2212,10 @@ export class ApplicationInsightsCoreTests extends AITestClass {
         });
 
         this.testCase({
-            name: "FieldRedaction: should redact custom query parameters defined in redactQueryParams",
+            name: "FieldRedaction: should redact custom query parameters defined in redactQueryParams and replace custom queryParams",
             test: () => {
                 let config = {
+                    redactUrls: UrlRedactionOptions.replace,
                     redactQueryParams: ["authorize", "api_key", "password"]
                 } as IConfiguration;
 
@@ -2213,6 +2229,7 @@ export class ApplicationInsightsCoreTests extends AITestClass {
             name: "FieldRedaction: should redact both default and custom query parameters",
             test: () => {
                 let config = {
+                    redactUrls: UrlRedactionOptions.append,
                     redactQueryParams: ["auth_token"]
                 } as IConfiguration;
 
@@ -2223,26 +2240,24 @@ export class ApplicationInsightsCoreTests extends AITestClass {
             }
         });
         this.testCase({
-            name: "FieldRedaction:should not redact custom parameters when redaction is disabled",
+            name: "FieldRedaction:should replace custom parameters redactQueryParams when user specifies the replace config",
             test: () => {
                 let config = {
-                    redactUrls: false,
+                    redactUrls: UrlRedactionOptions.replace,
                     redactQueryParams: ["authorize", "api_key"]
                 } as IConfiguration;
 
-                const url = "https://example.com/path?auth_token=12345&authorize=secret";
+                const url = "https://username:password@example.com/path?auth_token=12345&authorize=secret";
                 const redactedLocation = fieldRedaction(url, config);
-                Assert.equal(redactedLocation, "https://example.com/path?auth_token=12345&authorize=secret",
-                    "URL with custom sensitive parameters should not be redacted when redaction is disabled");
+                Assert.equal(redactedLocation, "https://REDACTED:REDACTED@example.com/path?auth_token=12345&authorize=REDACTED",
+                    "URL with custom sensitive parameters should be redacted when query redaction is not disabled");
             }
         });
 
         this.testCase({
             name: "FieldRedaction: should handle empty redactQueryParams array",
             test: () => {
-                let config = {
-                    redactQueryParams: []
-                } as IConfiguration;
+                let config = {} as IConfiguration;
 
                 // Should still redact default parameters
                 const url = "https://example.com/path?Signature=secret&custom_param=value";
@@ -2256,6 +2271,7 @@ export class ApplicationInsightsCoreTests extends AITestClass {
             name: "FieldRedaction:should handle complex URLs with both credentials and custom query parameters",
             test: () => {
                 let config = {
+                    redactUrls: UrlRedactionOptions.append,
                     redactQueryParams: ["authorize", "session_id"]
                 } as IConfiguration;
 
@@ -2584,6 +2600,34 @@ export class ApplicationInsightsCoreTests extends AITestClass {
             }
         });
 
+        this.testCase({
+                name: "FieldRedaction: should redact credentials while preserving query strings when redactQueryParams is false",
+            test: () => {
+                    let config = {
+                        redactUrls: 5
+                    } as IConfiguration;
+                    const url = "https://user:password@example.com/path?sig=secret&color=blue&token=abc123";
+                const redactedLocation = fieldRedaction(url, config);
+                    Assert.equal(redactedLocation, "https://REDACTED:REDACTED@example.com/path?sig=secret&color=blue&token=abc123",
+                        "Credentials should be redacted while query string values remain unchanged when redactQueryParams is false");
+            }
+        });
+
+        this.testCase({
+            name: "FieldRedaction: should handle custom parameters with multiple occurrences and empty values",
+            test: () => {
+                let config = {
+                    redactUrls: UrlRedactionOptions.replace,
+                    redactQueryParams: ["auth_token", "session_id"]
+                } as IConfiguration;
+                const url = "https://example.com/path?auth_token=first&name=test&auth_token=&session_id=abc&session_id=";
+                const redactedLocation = fieldRedaction(url, config);
+                // Only redact parameters that have actual values, not empty ones
+                Assert.equal(redactedLocation, "https://example.com/path?auth_token=REDACTED&name=test&auth_token=&session_id=REDACTED&session_id=",
+                    "Only non-empty custom sensitive parameters should be redacted");
+            }
+        });
+
         this.testCase({
             name: "FieldRedaction: should handle parameters without values mixed with valued parameters",
             test: () => {
@@ -2598,16 +2642,28 @@ export class ApplicationInsightsCoreTests extends AITestClass {
         });
 
         this.testCase({
-            name: "FieldRedaction: should handle custom parameters with multiple occurrences and empty values",
+            name: "FieldRedaction: should redact all parts of the URL (username, password, default query params) when redactUrls is set to True",
             test: () => {
                 let config = {
-                    redactQueryParams: ["auth_token", "session_id"]
+                    redactUrls: true
                 } as IConfiguration;
-                const url = "https://example.com/path?auth_token=first&name=test&auth_token=&session_id=abc&session_id=";
+                const url = "https://user:password@example.com/path?sig=secret&color=blue&token=abc123";
                 const redactedLocation = fieldRedaction(url, config);
-                // Only redact parameters that have actual values, not empty ones
-                Assert.equal(redactedLocation, "https://example.com/path?auth_token=REDACTED&name=test&auth_token=&session_id=REDACTED&session_id=",
-                    "Only non-empty custom sensitive parameters should be redacted");
+                Assert.equal(redactedLocation, "https://REDACTED:REDACTED@example.com/path?sig=REDACTED&color=blue&token=abc123",
+                "All parts of the URL should be redacted when redactUrls is true");
+            }
+        });
+
+        this.testCase({
+            name: "FieldRedaction: should not redact credentials or query strings when redactUrls and redactQueryParams are false",
+            test: () => {
+                let config = {
+                    redactUrls: UrlRedactionOptions.false
+                } as IConfiguration;
+                const url = "https://user:password@example.com/path?sig=secret&color=blue&token=abc123";
+                const redactedLocation = fieldRedaction(url, config);
+                Assert.equal(redactedLocation, url,
+                    "Nothing should be redacted when both redactUrls and redactQueryParams are false");
             }
         });
 

shared/AppInsightsCore/src/enums/ai/UrlRedactionOptions.ts

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+/**
+ * Controls how the user can configure which parts of the URL should be redacted. Example, certain query parameters, username and password, etc.
+ * @since <next_release_version>
+ */
+export const enum UrlRedactionOptions {
+    /**
+     * The default value, will redact the username and password as well as the default set of query parameters
+     */
+    true = 1,
+
+    /**
+     * Does not redact username and password or any query parameters, the URL will be left as is. Note: this is not recommended as it may lead 
+     * to sensitive data being sent in clear text.
+     */
+    false = 2,
+
+    /**
+     * This will append any additional queryParams that the user has provided through redactQueryParams config to the default set i.e to
+     * @defaultValue ["sig", "Signature", "AWSAccessKeyId", "X-Goog-Signature"]. 
+     */
+    append = 3,
+
+    /**
+     * This will replace the default set of query parameters to redact with the query parameters defined in redactQueryParams config, if provided by the user.
+     */
+    replace = 4,
+
+    /**
+     * This will redact username and password in the URL but will not redact any query parameters, even those in the default set.
+     */
+    usernamePasswordOnly = 5,
+
+    /**
+     * This will only redact the query parameter in the default set of query parameters to redact. It will not redact username and password.
+     */
+    queryParamsOnly = 6,
+
+}

shared/AppInsightsCore/src/interfaces/ai/IConfiguration.ts

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 import { IPromise } from "@nevware21/ts-async";
 import { eTraceHeadersMode } from "../../enums/ai/TraceHeadersMode";
+import { UrlRedactionOptions } from "../../enums/ai/UrlRedactionOptions";
 import { IOTelConfig } from "../otel/config/IOTelConfig";
 import { IAppInsightsCore } from "./IAppInsightsCore";
 import { IChannelControls } from "./IChannelControls";
@@ -232,10 +233,10 @@ export interface IConfiguration extends IOTelConfig {
     expCfg?: IExceptionConfig;
 
     /**
-     * [Optional] A flag to enable or disable the use of the field redaction for urls.
+     * [Optional] A flag to enable or disable redaction for query parameters.
      * @defaultValue true
      */
-    redactUrls?: boolean;
+    redactUrls?: boolean | UrlRedactionOptions;
 
     /**
      * [Optional] Additional query parameters to redact beyond the default set.

shared/AppInsightsCore/src/utils/EnvUtils.ts

@@ -8,6 +8,7 @@
     isFunction, isNullOrUndefined, isString, isUndefined, mathMax, strIndexOf, strSubstring
 } from "@nevware21/ts-utils";
 import { DEFAULT_SENSITIVE_PARAMS, STR_EMPTY, STR_REDACTED } from "../constants/InternalConstants";
+import { UrlRedactionOptions } from "../enums/ai/UrlRedactionOptions";
 import { IConfiguration } from "../interfaces/ai/IConfiguration";
 import { strContains } from "./HelperFuncs";
 
@@ -455,11 +456,13 @@
         return url;
     }
 
-    if (config && config.redactQueryParams) {
+    if (config && config.redactUrls === UrlRedactionOptions.append) {
         sensitiveParams = DEFAULT_SENSITIVE_PARAMS.concat(config.redactQueryParams);
+    } else if (config && config.redactUrls === UrlRedactionOptions.replace) {
+        sensitiveParams = config.redactQueryParams;
     } else {
         sensitiveParams = DEFAULT_SENSITIVE_PARAMS;
     }
 
     const baseUrl = strSubstring(url, 0, questionMarkIndex + 1);
     let queryString = strSubstring(url, questionMarkIndex + 1);
@@ -543,17 +546,27 @@
     if (!input || !isString(input) || strIndexOf(input, " ") !== -1) {
         return input;
     }
-    const isRedactionDisabled = config && config.redactUrls === false;
+    const isRedactionDisabled = config && (config.redactUrls === false || config.redactUrls === UrlRedactionOptions.false);
     if (isRedactionDisabled) {
         return input;
     }
-    const hasCredentials = strIndexOf(input, "@") !== -1;
-    const hasQueryParams = strIndexOf(input, "?") !== -1;
+
+    let hasCredentials = strIndexOf(input, "@") !== -1;
+    let hasQueryParams = strIndexOf(input, "?") !== -1;
 
     // If no credentials and no query params, return original
     if (!hasCredentials && !hasQueryParams) {
         return input;
     }
+
+    if (config.redactUrls === UrlRedactionOptions.usernamePasswordOnly) {
+        hasQueryParams = false;
+    }
+
+    if (config.redactUrls === UrlRedactionOptions.queryParamsOnly) {
+        hasCredentials = false;
+    }
+
     try {
         let result = input;
         if (hasCredentials) {