feat: add redactMessage function to sanitize sensitive data in logs

Author: qevolg

nodejs/src/common/log.ts

@@ -1,6 +1,27 @@
 import winston from "winston";
 import DailyRotateFile from "winston-daily-rotate-file";
 
+export function redactMessage(msg: any): any {
+    if (typeof msg === "string") {
+        return msg
+            .replace(/("password"\s*:\s*")([^"]*)(")/gi, '$1[REDACTED]$3')
+            .replace(/(token=)([^&\s]+)/gi, "$1[REDACTED]")
+            .replace(/(\/\/[^:@\s]+:)([^@/\s]+)(@)/gi, "$1[REDACTED]$3");
+    }
+    if (msg && typeof msg === "object") {
+        const clone: any = Array.isArray(msg) ? [] : {};
+        for (const [k, v] of Object.entries(msg)) {
+            if (["password", "token", "bearer_token"].includes(k.toLowerCase())) {
+                clone[k] = "[REDACTED]";
+            } else {
+                clone[k] = redactMessage(v);
+            }
+        }
+        return clone;
+    }
+    return msg;
+}
+
 const customFormat = winston.format.printf(
     ({ level, message, label, timestamp }) => {
         if (
@@ -10,6 +31,7 @@ const customFormat = winston.format.printf(
         ) {
             message = (message as any).toJSON();
         }
+        message = redactMessage(message);
         return `${timestamp} [${label}] ${level}: ${message}`;
     }
 );

nodejs/src/tmq/tmqResponse.ts

@@ -147,11 +147,8 @@ export class WSTmqFetchBlockInfo {
         this.withTableName = dataView.getUint8(4) == 1 ? true : false;
         this.withSchema = dataView.getUint8(5) == 1 ? true : false;
 
-        // let dataBuffer = dataView.buffer.slice(6)
         let dataBuffer = new DataView(dataView.buffer, dataView.byteOffset + 6);
         let rows = 0;
-        // const parseStartTime = new Date().getTime();
-        // console.log("parseBlockInfos blockNum="+ blockNum)
         for (let i = 0; i < blockNum; i++) {
             let variableInfo = this.parseVariableByteInteger(dataBuffer);
             this.taosResult.setPrecision(variableInfo[1].getUint8(17));

nodejs/test/bulkPulling/log.test.ts

@@ -1,4 +1,4 @@
-import logger, { setLevel } from "../../src/common/log";
+import logger, { setLevel, redactMessage } from "../../src/common/log";
 
 describe("log level print", () => {
     test("normal connect", async () => {
@@ -21,3 +21,103 @@ describe("log level print", () => {
         expect(isLevel).toEqual(true);
     });
 });
+
+describe("redact message", () => {
+    test("redacts password field in JSON-like string", () => {
+        const input = '{"password": "secret123"}';
+        const output = redactMessage(input);
+        expect(output).toBe('{"password": "[REDACTED]"}');
+    });
+
+    test("redacts token in query string", () => {
+        const input = "https://example.com?token=abcdef123456&other=1";
+        const output = redactMessage(input);
+        expect(output).toBe(
+            "https://example.com?token=[REDACTED]&other=1"
+        );
+    });
+
+    test("redacts bearer token in query string", () => {
+        const input = "https://example.com?bearer_token=abcdef123456&other=1";
+        const output = redactMessage(input);
+        expect(output).toBe(
+            "https://example.com?bearer_token=[REDACTED]&other=1"
+        );
+    });
+
+    test("is case-insensitive for password key", () => {
+        const input = '{"PassWord": "secret123"}';
+        const output = redactMessage(input);
+        expect(output).toBe('{"PassWord": "[REDACTED]"}');
+    });
+
+    test("leaves string without sensitive data unchanged", () => {
+        const input = "normal message without secrets";
+        const output = redactMessage(input);
+        expect(output).toBe(input);
+    });
+
+    test("redacts password and token fields on plain object", () => {
+        const input = {
+            user: "u1",
+            password: "secret",
+            token: "abc123",
+            bearer_token: "def456",
+            other: "keep",
+        };
+        const output = redactMessage(input) as any;
+
+        expect(output.user).toBe("u1");
+        expect(output.password).toBe("[REDACTED]");
+        expect(output.token).toBe("[REDACTED]");
+        expect(output.bearer_token).toBe("[REDACTED]");
+        expect(output.other).toBe("keep");
+    });
+
+    test("redacts nested objects recursively", () => {
+        const input = {
+            level1: {
+                password: "p1",
+                nested: {
+                    token: "t1",
+                    value: 42,
+                },
+            },
+        };
+        const output = redactMessage(input) as any;
+
+        expect(output.level1.password).toBe("[REDACTED]");
+        expect(output.level1.nested.token).toBe("[REDACTED]");
+        expect(output.level1.nested.value).toBe(42);
+    });
+
+    test("redacts objects inside array", () => {
+        const input = [
+            { password: "p1" },
+            { token: "t2", ok: true },
+        ];
+        const output = redactMessage(input) as any[];
+
+        expect(output[0].password).toBe("[REDACTED]");
+        expect(output[1].token).toBe("[REDACTED]");
+        expect(output[1].ok).toBe(true);
+    });
+
+    test("returns same primitive for non-object/non-string", () => {
+        expect(redactMessage(123)).toBe(123);
+        expect(redactMessage(null)).toBeNull();
+        expect(redactMessage(undefined)).toBeUndefined();
+    });
+
+    test("redacts password in ws url user:password@", () => {
+        const input = "ws://root:taosdata@localhost:6041/ws";
+        const output = redactMessage(input);
+        expect(output).toBe("ws://root:[REDACTED]@localhost:6041/ws");
+    });
+
+    test("does not change url without credentials", () => {
+        const input = "ws://localhost:6041/ws";
+        const output = redactMessage(input);
+        expect(output).toBe(input);
+    });
+});