CLDSRV-750: format server access logs to s.ms

Author: leif-scality

lib/utilities/serverAccessLogger.js

@@ -358,6 +358,15 @@ function calculateElapsedMS(startTime, onCloseEndTime) {
     return Number(onCloseEndTime - startTime) / 1_000_000;
 }
 
+function timestampToDateTime643(unixMS) {
+    if (unixMS === undefined || unixMS === null) {
+        return null;
+    }
+
+    // clickhouse DateTime64(3) expects "seconds.milliseconds" (string type).
+    return (unixMS / 1000).toFixed(3);
+}
+
 function logServerAccess(req, res) {
     if (!req.serverAccessLog || !res.serverAccessLog || !serverAccessLogger) {
         return;
@@ -372,7 +381,8 @@ function logServerAccess(req, res) {
 
     serverAccessLogger.write(`${JSON.stringify({
         // Werelog fields
-        time: Date.now(),
+        // logs.access_logs_ingest.timestamp in Clickhouse is of type DateTime so it expects seconds as an integer.
+        time: Math.floor(Date.now() / 1000),
         hostname,
         pid: process.pid,
 
@@ -392,7 +402,7 @@ function logServerAccess(req, res) {
         httpURL: req.url || null,
 
         // AWS access server logs fields https://docs.aws.amazon.com/AmazonS3/latest/userguide/LogFormat.html
-        startTime: params.startTimeUnixMS || null, // AWS "Time" field
+        startTime: timestampToDateTime643(params.startTimeUnixMS), // AWS "Time" field
         requester: getRequester(authInfo),
         operation: getOperation(req),
         requestURI: getURI(req),
@@ -446,4 +456,5 @@ module.exports = {
     getBytesSent,
     calculateTotalTime,
     calculateTurnAroundTime,
+    timestampToDateTime643,
 };

tests/unit/utils/serverAccessLogger.js

@@ -11,6 +11,7 @@ const {
     getBytesSent,
     calculateTotalTime,
     calculateTurnAroundTime,
+    timestampToDateTime643,
 } = require('../../../lib/utilities/serverAccessLogger');
 
 
@@ -618,6 +619,48 @@ describe('serverAccessLogger utility functions', () => {
         });
     });
 
+    describe('timestampToDateTime643', () => {
+        it('should convert milliseconds to seconds with 3 decimal places', () => {
+            const startTimeUnixMS = 1234567890000;
+            const result = timestampToDateTime643(startTimeUnixMS);
+            assert.strictEqual(result, '1234567890.000');
+        });
+
+        it('should handle timestamp with milliseconds', () => {
+            const startTimeUnixMS = 1234567890123;
+            const result = timestampToDateTime643(startTimeUnixMS);
+            assert.strictEqual(result, '1234567890.123');
+        });
+
+        it('should handle 0', () => {
+            const startTimeUnixMS = 0;
+            const result = timestampToDateTime643(startTimeUnixMS);
+            assert.strictEqual(result, '0.000');
+        });
+
+        it('should return null when startTimeUnixMS is null', () => {
+            const result = timestampToDateTime643(null);
+            assert.strictEqual(result, null);
+        });
+
+        it('should return null when startTimeUnixMS is undefined', () => {
+            const result = timestampToDateTime643(undefined);
+            assert.strictEqual(result, null);
+        });
+
+        it('should handle small timestamps', () => {
+            const startTimeUnixMS = 1000;
+            const result = timestampToDateTime643(startTimeUnixMS);
+            assert.strictEqual(result, '1.000');
+        });
+
+        it('should handle timestamps with partial milliseconds', () => {
+            const startTimeUnixMS = 1500;
+            const result = timestampToDateTime643(startTimeUnixMS);
+            assert.strictEqual(result, '1.500');
+        });
+    });
+
     describe('logServerAccess', () => {
         let mockLogger;
         let sandbox;
@@ -773,7 +816,7 @@ describe('serverAccessLogger utility functions', () => {
             assert.strictEqual(loggedData.httpURL, '/test-bucket/test-key.txt');
 
             // Verify AWS access server log fields
-            assert.strictEqual(loggedData.startTime, 1234567890000);
+            assert.strictEqual(loggedData.startTime, '1234567890.000');
             assert.strictEqual(loggedData.requester, 'canonical123');
             assert.strictEqual(loggedData.operation, 'REST.GET.OBJECT');
             assert.strictEqual(loggedData.requestURI, 'GET /test-bucket/test-key.txt HTTP/1.1');