Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/wet-stingrays-behave.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
"@hyperdx/api": minor
"@hyperdx/app": minor
---

migration: migrate to Pino for standardized and faster logging
7 changes: 4 additions & 3 deletions packages/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"@ai-sdk/anthropic": "^2.0.23",
"@esm2cjs/p-queue": "^7.3.0",
"@hyperdx/common-utils": "^0.7.0",
"@hyperdx/node-opentelemetry": "^0.8.2",
"@hyperdx/node-opentelemetry": "^0.9.0",
"@hyperdx/passport-local-mongoose": "^9.0.1",
"@opentelemetry/api": "^1.8.0",
"@opentelemetry/host-metrics": "^0.35.5",
Expand All @@ -27,7 +27,6 @@
"express": "^4.19.2",
"express-rate-limit": "^6.7.1",
"express-session": "^1.17.3",
"express-winston": "^4.2.0",
"handlebars": "^4.7.8",
"http-graceful-shutdown": "^3.1.13",
"http-proxy-middleware": "^3.0.5",
Expand All @@ -40,12 +39,13 @@
"on-headers": "^1.1.0",
"passport": "^0.6.0",
"passport-local": "^1.0.0",
"pino": "^10.0.0",
"pino-http": "^11.0.0",
"promised-handlebars": "^2.0.1",
"protobufjs": "^7.5.2",
"semver": "^7.5.2",
"serialize-error": "^8.1.0",
"uuid": "^8.3.2",
"winston": "^3.10.0",
"zod": "3.25",
"zod-express-middleware": "^1.4.0"
},
Expand All @@ -67,6 +67,7 @@
"jest": "^28.1.3",
"migrate-mongo": "^11.0.0",
"nodemon": "^2.0.20",
"pino-pretty": "^13.1.1",
"rimraf": "^4.4.1",
"supertest": "^6.3.1",
"swagger-jsdoc": "^6.2.8",
Expand Down
8 changes: 5 additions & 3 deletions packages/api/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ if (config.IS_DEV) {
const server = new Server();

process.on('uncaughtException', (err: Error) => {
logger.error(serializeError(err));
logger.error({ err: serializeError(err) }, 'Uncaught exception');

// FIXME: disable server restart until
// we make sure all expected exceptions are handled properly
Expand All @@ -32,7 +32,9 @@ process.on('uncaughtException', (err: Error) => {

process.on('unhandledRejection', (err: any) => {
// TODO: do we want to throw here ?
logger.error(serializeError(err));
logger.error({ err: serializeError(err) }, 'Unhandled rejection');
});

server.start().catch(e => logger.error(serializeError(e)));
server
.start()
.catch(e => logger.error({ err: serializeError(e) }, 'Server start failed'));
3 changes: 2 additions & 1 deletion packages/api/src/middleware/auth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ export function redirectToDashboard(req: Request, res: Response) {
return res.redirect(`${config.FRONTEND_URL}/search`);
} else {
logger.error(
`Password login for user failed, user or team not found ${req?.user?._id}`,
{ userId: req?.user?._id },
'Password login for user failed, user or team not found',
);
res.redirect(`${config.FRONTEND_URL}/login?err=unknown`);
}
Expand Down
6 changes: 3 additions & 3 deletions packages/api/src/models/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ mongoose.connection.on('disconnected', () => {
logger.info('Lost connection to MongoDB server');
});

mongoose.connection.on('error', () => {
logger.error('Could not connect to MongoDB');
mongoose.connection.on('error', err => {
logger.error({ err }, 'Could not connect to MongoDB');
});

mongoose.connection.on('reconnected', () => {
logger.error('Reconnected to MongoDB');
logger.warn('Reconnected to MongoDB');
});

mongoose.connection.on('reconnectFailed', () => {
Expand Down
4 changes: 2 additions & 2 deletions packages/api/src/opamp/controllers/opampController.ts
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ export class OpampController {

// Decode the AgentToServer message
const agentToServer = decodeAgentToServer(req.body);
logger.debug('agentToServer', agentToServer);
logger.debug({ agentToServer }, 'agentToServer');
logger.debug(
// @ts-ignore
`Received message from agent: ${agentToServer.instanceUid?.toString(
Expand Down Expand Up @@ -353,7 +353,7 @@ export class OpampController {
res.setHeader('Content-Type', 'application/x-protobuf');
res.send(encodedResponse);
} catch (error) {
logger.error('Error handling OpAMP message:', error);
logger.error({ err: error }, 'Error handling OpAMP message');
res.status(500).send('Internal Server Error');
}
}
Expand Down
2 changes: 1 addition & 1 deletion packages/api/src/opamp/services/agentService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ export class AgentService {

return agent;
} catch (error) {
logger.error('Error processing agent status:', error);
logger.error({ err: error }, 'Error processing agent status');
throw error;
}
}
Expand Down
10 changes: 5 additions & 5 deletions packages/api/src/opamp/utils/protobuf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ try {
root = protobuf.loadSync(PROTO_PATH);
logger.debug('OpAMP proto definition loaded successfully');
} catch (error) {
logger.error('Failed to load OpAMP proto definition:', error);
logger.error({ err: error }, 'Failed to load OpAMP proto definition');
throw error;
}

Expand All @@ -46,7 +46,7 @@ export function decodeAgentToServer(data: Buffer): protobuf.Message {
try {
return AgentToServer.decode(data);
} catch (error) {
logger.error('Failed to decode AgentToServer message:', error);
logger.error({ err: error }, 'Failed to decode AgentToServer message');
throw error;
}
}
Expand All @@ -68,7 +68,7 @@ export function encodeServerToAgent(message: any): Buffer {
// Encode the message
return Buffer.from(ServerToAgent.encode(serverToAgent).finish());
} catch (error) {
logger.error('Failed to encode ServerToAgent message:', error);
logger.error({ err: error }, 'Failed to encode ServerToAgent message');
throw error;
}
}
Expand Down Expand Up @@ -105,7 +105,7 @@ export function createRemoteConfig(
configHash: configHash,
};
} catch (error) {
logger.error('Failed to create remote config message:', error);
logger.error({ err: error }, 'Failed to create remote config message');
throw error;
}
}
Expand All @@ -130,7 +130,7 @@ function calculateConfigHash(configFiles: Map<string, Buffer>): Buffer {

return hash.digest();
} catch (error) {
logger.error('Failed to calculate config hash:', error);
logger.error({ err: error }, 'Failed to calculate config hash');
throw error;
}
}
Expand Down
15 changes: 10 additions & 5 deletions packages/api/src/routers/api/root.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,10 @@ router.post(
password,
async (err: Error, user: any) => {
if (err) {
logger.error(serializeError(err));
logger.error(
{ err: serializeError(err) },
'User registration error',
);
return res.status(400).json({ error: 'invalid' });
}

Expand All @@ -107,7 +110,8 @@ router.post(
await setupTeamDefaults(team._id.toString());
} catch (error) {
logger.error(
`Failed to setup team defaults: ${serializeError(error)}`,
{ err: serializeError(error) },
'Failed to setup team defaults',
);
// Continue with registration even if setup defaults fails
}
Expand All @@ -118,7 +122,8 @@ router.post(
}

logger.error(
`Password login for user failed, user or team not found ${req?.user?._id}`,
{ userId: req?.user?._id },
'Password login for user failed, user or team not found',
);
return res.status(400).json({ error: 'invalid' });
});
Expand Down Expand Up @@ -167,7 +172,7 @@ router.post('/team/setup/:token', async (req, res, next) => {
password, // TODO: validate password
async (err: Error, user: any) => {
if (err) {
logger.error(serializeError(err));
logger.error({ err: serializeError(err) }, 'Team setup error');
return res.redirect(
`${config.FRONTEND_URL}/join-team?token=${token}&err=500`,
);
Expand Down Expand Up @@ -196,7 +201,7 @@ router.get('/ext/silence-alert/:token', async (req, res) => {
await silenceAlertByToken(token);
} catch (e) {
isError = true;
logger.error(e);
logger.error({ err: e }, 'Failed to silence alert');
}

// TODO: Create a template for utility pages
Expand Down
5 changes: 4 additions & 1 deletion packages/api/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@ export default class Server {

if (mongoCloseResult.status === 'rejected') {
hasError = true;
logger.error(serializeError(mongoCloseResult.reason));
logger.error(
{ err: serializeError(mongoCloseResult.reason) },
'MongoDB client close failed',
);
} else {
logger.info('MongoDB client closed.');
}
Expand Down
10 changes: 5 additions & 5 deletions packages/api/src/setupDefaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ export async function setupTeamDefaults(teamId: string) {
// Get the team object
const team = await getTeam(teamId);
if (!team) {
logger.warn(`Team not found with ID: ${teamId}`);
logger.warn({ teamId }, 'Team not found');
return;
}

Expand Down Expand Up @@ -71,7 +71,7 @@ export async function setupTeamDefaults(teamId: string) {
`Created default connection: ${connectionConfig.name} (${newConnection._id})`,
);
} catch (error) {
logger.error(`Failed to create connection: ${error}`);
logger.error({ err: error }, 'Failed to create connection');
}
}
} else if (parsedDefaultConnections) {
Expand Down Expand Up @@ -136,7 +136,7 @@ export async function setupTeamDefaults(teamId: string) {
c => c.name === connectionId,
);
if (!connection) {
logger.warn(`Connection not found with name: ${connectionId}`);
logger.warn({ connectionId }, 'Connection not found with name');
continue;
}
connectionId = connection._id.toString();
Expand Down Expand Up @@ -166,7 +166,7 @@ export async function setupTeamDefaults(teamId: string) {
// Store the created source for the second pass
createdSources[sourceConfig.name] = newSource;
} catch (error) {
logger.error(`Failed to create source: ${error}`);
logger.error({ err: error }, 'Failed to create source');
}
}

Expand Down Expand Up @@ -227,7 +227,7 @@ export async function setupTeamDefaults(teamId: string) {
);
}
} catch (error) {
logger.error(`Failed to update source references: ${error}`);
logger.error({ err: error }, 'Failed to update source references');
}
}
} else if (parsedDefaultSources) {
Expand Down
6 changes: 3 additions & 3 deletions packages/api/src/tasks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,19 +70,19 @@ if (!RUN_SCHEDULED_TASKS_EXTERNALLY) {
})
.catch(err => {
console.log(err);
logger.error(serializeError(err));
logger.error({ err: serializeError(err) }, 'Task execution failed');
process.exit(1);
});
}

process.on('uncaughtException', (err: Error) => {
console.log(err);
logger.error(serializeError(err));
logger.error({ err: serializeError(err) }, 'Uncaught exception');
process.exit(1);
});

process.on('unhandledRejection', (err: any) => {
console.log(err);
logger.error(serializeError(err));
logger.error({ err: serializeError(err) }, 'Unhandled rejection');
process.exit(1);
});
11 changes: 7 additions & 4 deletions packages/api/src/tasks/template.ts
Original file line number Diff line number Diff line change
Expand Up @@ -347,15 +347,18 @@ const getPopulatedChannel = (
findWebhookByName(channelIdOrNamePrefix, teamWebhooksById);

if (!webhook) {
logger.error('webhook not found', {
webhookId: channelIdOrNamePrefix,
});
logger.error(
{
webhookId: channelIdOrNamePrefix,
},
'webhook not found',
);
return undefined;
}
return { type: 'webhook', channel: webhook };
}
default: {
logger.error(`unsupported alert channel type: ${channelType}`);
logger.error({ channelType }, 'Unsupported alert channel type');
return undefined;
}
}
Expand Down
26 changes: 13 additions & 13 deletions packages/api/src/tasks/usageStats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,27 @@ import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types';
import * as HyperDX from '@hyperdx/node-opentelemetry';
import ms from 'ms';
import os from 'os';
import winston from 'winston';
import pino from 'pino';

import * as config from '@/config';
import Connection from '@/models/connection';
import { Source, SourceDocument } from '@/models/source';
import Team from '@/models/team';
import User from '@/models/user';

const logger = winston.createLogger({
const logger = pino({
level: 'info',
format: winston.format.json(),
transports: [
HyperDX.getWinstonTransport('info', {
headers: {
Authorization: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key
},
baseUrl: 'https://in-otel.hyperdx.io/v1/logs',
maxLevel: 'info',
service: 'hyperdx-oss-usage-stats',
}),
],
transport: {
targets: [
HyperDX.getPinoTransport('info', {
headers: {
Authorization: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key
},
baseUrl: 'https://in-otel.hyperdx.io/v1/logs',
service: 'hyperdx-oss-usage-stats',
}),
],
},
});

function extractTableNames(source: SourceDocument): string[] {
Expand Down
Loading
Loading