Replace Amplitude with PostHog for product metrics (#227) (#237)

vcarl · claude · web-flow · commit 374f8d08f84b · 2026-01-06T15:40:36.000-05:00
Migrates backend analytics from Amplitude HTTP API to PostHog Node SDK. Adds new feature interaction events for tickets, honeypot, reactji, spam detection, and bot install/uninstall tracking. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
diff --git a/.env.example b/.env.example
@@ -23,5 +23,9 @@ DIGITALOCEAN_TOKEN=
 SENTRY_INGEST=
 SENTRY_RELEASES=
 
+## PostHog analytics (frontend)
 VITE_PUBLIC_POSTHOG_KEY=phc_…
 VITE_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com
+## PostHog analytics (backend - can use same key as frontend)
+POSTHOG_KEY=phc_…
+POSTHOG_HOST=https://us.i.posthog.com
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -132,6 +132,8 @@ jobs:
             STRIPE_WEBHOOK_SECRET: "${{ secrets.STRIPE_WEBHOOK_SECRET }}"
             VITE_PUBLIC_POSTHOG_KEY: "${{ secrets.VITE_PUBLIC_POSTHOG_KEY }}"
             VITE_PUBLIC_POSTHOG_HOST: "${{ secrets.VITE_PUBLIC_POSTHOG_HOST }}"
+            POSTHOG_KEY: "${{ secrets.VITE_PUBLIC_POSTHOG_KEY }}"
+            POSTHOG_HOST: "${{ secrets.VITE_PUBLIC_POSTHOG_HOST }}"
             DATABASE_URL: "${{ secrets.DATABASE_URL }}"
           EOF
 
@@ -179,6 +181,8 @@ jobs:
             --from-literal=STRIPE_WEBHOOK_SECRET=${{ secrets.STRIPE_WEBHOOK_SECRET }} \
             --from-literal=VITE_PUBLIC_POSTHOG_KEY=${{ secrets.VITE_PUBLIC_POSTHOG_KEY }} \
             --from-literal=VITE_PUBLIC_POSTHOG_HOST=${{ secrets.VITE_PUBLIC_POSTHOG_HOST }} \
+            --from-literal=POSTHOG_KEY=${{ secrets.VITE_PUBLIC_POSTHOG_KEY }} \
+            --from-literal=POSTHOG_HOST=${{ secrets.VITE_PUBLIC_POSTHOG_HOST }} \
             --from-literal=DATABASE_URL=/data/mod-bot.sqlite3 \
             --dry-run=client -o yaml | kubectl apply -f -
 
diff --git a/app/commands/setupHoneypot.ts b/app/commands/setupHoneypot.ts
@@ -8,6 +8,7 @@ import {
 
 import db from "#~/db.server.js";
 import type { AnyCommand } from "#~/helpers/discord.js";
+import { featureStats } from "#~/helpers/metrics";
 import { log } from "#~/helpers/observability.js";
 
 const DEFAULT_MESSAGE_TEXT =
@@ -65,6 +66,11 @@ export const Command = [
           .execute();
         if (result[0].numInsertedOrUpdatedRows ?? 0 > 0) {
           await castedChannel.send(messageText);
+          featureStats.honeypotSetup(
+            interaction.guildId,
+            interaction.user.id,
+            honeypotChannel.id,
+          );
         }
 
         await interaction.reply({
diff --git a/app/commands/setupReactjiChannel.ts b/app/commands/setupReactjiChannel.ts
@@ -7,6 +7,7 @@ import {
 
 import db from "#~/db.server.js";
 import { type SlashCommand } from "#~/helpers/discord";
+import { featureStats } from "#~/helpers/metrics";
 
 export const Command = {
   command: new SlashCommandBuilder()
@@ -86,6 +87,13 @@ export const Command = {
         )
         .execute();
 
+      featureStats.reactjiChannelSetup(
+        guildId,
+        configuredById,
+        emoji,
+        threshold,
+      );
+
       const thresholdText =
         threshold === 1 ? "" : ` (after ${threshold} reactions)`;
       await interaction.reply({
diff --git a/app/commands/setupTickets.ts b/app/commands/setupTickets.ts
@@ -24,6 +24,7 @@ import {
   type ModalCommand,
   type SlashCommand,
 } from "#~/helpers/discord";
+import { featureStats } from "#~/helpers/metrics";
 import { fetchSettings, SETTINGS } from "#~/models/guilds.server";
 
 const DEFAULT_BUTTON_TEXT = "Open a private ticket with the moderators";
@@ -111,6 +112,12 @@ export const Command = [
             role_id: roleId,
           })
           .execute();
+
+        featureStats.ticketChannelSetup(
+          interaction.guild.id,
+          interaction.user.id,
+          ticketChannel?.id ?? interaction.channelId,
+        );
       } catch (e) {
         console.error(`error:`, e);
       }
@@ -204,7 +211,7 @@ export const Command = [
       await thread.send(`${user.displayName} said:
 ${quoteMessageContent(concern)}`);
       await thread.send({
-        content: "When you’ve finished, please close the ticket.",
+        content: "When you've finished, please close the ticket.",
         components: [
           // @ts-expect-error Types for this are super busted
           new ActionRowBuilder().addComponents(
@@ -224,6 +231,8 @@ ${quoteMessageContent(concern)}`);
         ],
       });
 
+      featureStats.ticketCreated(interaction.guild.id, user.id, thread.id);
+
       void interaction.reply({
         content: `A private thread with the moderation team has been opened for you: <#${thread.id}>`,
         ephemeral: true,
@@ -260,7 +269,7 @@ ${quoteMessageContent(concern)}`);
         rest.delete(Routes.threadMembers(threadId, ticketOpenerUserId)),
         rest.post(Routes.channelMessages(modLog), {
           body: {
-            content: `<@${ticketOpenerUserId}>’s ticket <#${threadId}> closed by <@${interactionUserId}>${feedback ? `. feedback: ${feedback}` : ""}`,
+            content: `<@${ticketOpenerUserId}>'s ticket <#${threadId}> closed by <@${interactionUserId}>${feedback ? `. feedback: ${feedback}` : ""}`,
             allowedMentions: {},
           },
         }),
@@ -270,6 +279,13 @@ ${quoteMessageContent(concern)}`);
         }),
       ]);
 
+      featureStats.ticketClosed(
+        interaction.guild.id,
+        interactionUserId,
+        ticketOpenerUserId,
+        !!feedback?.trim(),
+      );
+
       return;
     },
   } as MessageComponentCommand,
diff --git a/app/commands/track.tsx b/app/commands/track.tsx
@@ -7,6 +7,7 @@ import {
 import { Button } from "reacord";
 
 import { reacord } from "#~/discord/client.server";
+import { featureStats } from "#~/helpers/metrics";
 import { reportUser } from "#~/helpers/modLog";
 import {
   markMessageAsDeleted,
@@ -27,6 +28,10 @@ const handler = async (interaction: MessageContextMenuCommandInteraction) => {
     staff: user,
   });
 
+  if (interaction.guildId) {
+    featureStats.userTracked(interaction.guildId, user.id, message.author.id);
+  }
+
   const instance = reacord.ephemeralReply(
     interaction,
     <>
diff --git a/app/discord/automod.ts b/app/discord/automod.ts
@@ -2,6 +2,7 @@ import { Events, type Client } from "discord.js";
 
 import { isStaff } from "#~/helpers/discord";
 import { isSpam } from "#~/helpers/isSpam";
+import { featureStats } from "#~/helpers/metrics";
 import { reportUser } from "#~/helpers/modLog";
 import {
   markMessageAsDeleted,
@@ -34,6 +35,12 @@ export default async (bot: Client) => {
         .delete()
         .then(() => markMessageAsDeleted(message.id, message.guild!.id));
 
+      featureStats.spamDetected(
+        message.guild.id,
+        message.author.id,
+        message.channelId,
+      );
+
       if (warnings >= AUTO_SPAM_THRESHOLD) {
         await Promise.all([
           member.kick("Autokicked for spamming"),
@@ -42,6 +49,7 @@ export default async (bot: Client) => {
             allowedMentions: {},
           }),
         ]);
+        featureStats.spamKicked(message.guild.id, message.author.id, warnings);
       }
     }
   });
diff --git a/app/discord/deployCommands.server.ts b/app/discord/deployCommands.server.ts
@@ -42,6 +42,10 @@ export const deployCommands = async (client: Client) => {
     : deployTestCommands(client, localCommands));
 
   client.on(Events.InteractionCreate, (interaction) => {
+    log("info", "deployCommands", "Handling interaction", {
+      type: interaction.type,
+      id: interaction.id,
+    });
     switch (interaction.type) {
       case InteractionType.ApplicationCommand: {
         const config = matchCommand(interaction.commandName);
diff --git a/app/discord/gateway.ts b/app/discord/gateway.ts
@@ -7,7 +7,7 @@ import { deployCommands } from "#~/discord/deployCommands.server";
 import { startEscalationResolver } from "#~/discord/escalationResolver";
 import onboardGuild from "#~/discord/onboardGuild";
 import { startReactjiChanneler } from "#~/discord/reactjiChanneler";
-import { botStats } from "#~/helpers/metrics";
+import { botStats, shutdownMetrics } from "#~/helpers/metrics";
 import { log, trackPerformance } from "#~/helpers/observability";
 import Sentry from "#~/helpers/sentry.server";
 
@@ -134,4 +134,14 @@ export default function init() {
     // Track reconnections in business analytics
     botStats.reconnection(client.guilds.cache.size, client.users.cache.size);
   });
+
+  // Graceful shutdown handler to flush metrics
+  const handleShutdown = async (signal: string) => {
+    log("info", "Gateway", `Received ${signal}, shutting down gracefully`, {});
+    await shutdownMetrics();
+    process.exit(0);
+  };
+
+  process.on("SIGTERM", () => void handleShutdown("SIGTERM"));
+  process.on("SIGINT", () => void handleShutdown("SIGINT"));
 }
diff --git a/app/discord/honeypotTracker.ts b/app/discord/honeypotTracker.ts
@@ -1,6 +1,7 @@
 import { ChannelType, Events, type Client } from "discord.js";
 
 import db from "#~/db.server.js";
+import { featureStats } from "#~/helpers/metrics";
 import { reportUser } from "#~/helpers/modLog.js";
 import { log } from "#~/helpers/observability";
 import { fetchSettings, SETTINGS } from "#~/models/guilds.server.js";
@@ -103,6 +104,7 @@ export async function startHoneypotTracking(client: Client) {
           staff: client.user ?? false,
         }),
       ]);
+      featureStats.honeypotTriggered(msg.guildId, member.id, msg.channelId);
     } catch (e) {
       log(
         "error",
diff --git a/app/discord/onboardGuild.ts b/app/discord/onboardGuild.ts
@@ -1,5 +1,6 @@
 import { ChannelType, Events, type Client, type TextChannel } from "discord.js";
 
+import { botStats } from "#~/helpers/metrics";
 import { retry } from "#~/helpers/misc";
 import { fetchGuild } from "#~/models/guilds.server";
 
@@ -11,35 +12,49 @@ export default async (bot: Client) => {
   // available after downtime, or when actually added to a new guild
   bot.on(Events.GuildCreate, async (guild) => {
     const appGuild = await fetchGuild(guild.id);
-    if (!appGuild) {
-      await deployCommands(client);
-
-      const welcomeMessage = `Euno is here! Set it up with \`/setup\``;
-
-      const channels = await guild.channels.fetch();
-      const likelyChannels = channels.filter((c): c is TextChannel =>
-        Boolean(
-          c &&
-            c.type === ChannelType.GuildText &&
-            (c.name.includes("mod") || c.name.includes("intro")),
-        ),
-      );
-
-      await retry(5, async (n) => {
-        switch (n) {
-          case 0:
-            void guild.systemChannel!.send(welcomeMessage);
-            return;
-          case 1:
-            void guild.publicUpdatesChannel!.send(welcomeMessage);
-            return;
-          default: {
-            if (likelyChannels.size < n - 2) return;
-            void likelyChannels.at(n - 2)!.send(welcomeMessage);
-            return;
-          }
+    botStats.guildJoined(guild);
+    if (appGuild) return;
+
+    // New installation - deploy commands and send welcome message
+    await deployCommands(client);
+
+    const welcomeMessage = `Euno is here! Set it up with \`/setup\``;
+
+    const channels = await guild.channels.fetch();
+    const likelyChannels = channels.filter((c): c is TextChannel =>
+      Boolean(
+        c &&
+          c.type === ChannelType.GuildText &&
+          (c.name.includes("mod") || c.name.includes("intro")),
+      ),
+    );
+
+    await retry(5, async (n) => {
+      switch (n) {
+        case 0:
+          void guild.systemChannel!.send(welcomeMessage);
+          return;
+        case 1:
+          void guild.publicUpdatesChannel!.send(welcomeMessage);
+          return;
+        default: {
+          if (likelyChannels.size < n - 2) return;
+          void likelyChannels.at(n - 2)!.send(welcomeMessage);
+          return;
         }
-      });
-    }
+      }
+    });
+  });
+
+  // Track when the bot is removed from a guild
+  // GuildDelete also fires when a guild becomes temporarily unavailable,
+  // so we check the unavailable flag and verify the guild exists in our DB
+  bot.on(Events.GuildDelete, async (guild) => {
+    if (guild.available === false) return;
+
+    const appGuild = await fetchGuild(guild.id);
+    if (!appGuild) return;
+
+    botStats.guildRemoved(guild);
   });
 };
diff --git a/app/discord/reactjiChanneler.ts b/app/discord/reactjiChanneler.ts
@@ -1,6 +1,7 @@
 import { Events, type Client } from "discord.js";
 
 import db from "#~/db.server";
+import { featureStats } from "#~/helpers/metrics";
 import { log } from "#~/helpers/observability";
 
 export async function startReactjiChanneler(client: Client) {
@@ -100,6 +101,8 @@ export async function startReactjiChanneler(client: Client) {
         allowedMentions: { users: [] },
       });
 
+      featureStats.reactjiTriggered(guildId, user.id, emoji, message.id);
+
       log("info", "ReactjiChanneler", "Message forwarded successfully", {
         messageId: message.id,
         targetChannelId: config.channel_id,
diff --git a/app/helpers/env.server.ts b/app/helpers/env.server.ts
@@ -39,7 +39,8 @@ export const sentryReleases = getEnv("SENTRY_RELEASES", true);
 export const stripeSecretKey = getEnv("STRIPE_SECRET_KEY");
 export const stripeWebhookSecret = getEnv("STRIPE_WEBHOOK_SECRET");
 
-export const amplitudeKey = getEnv("AMPLITUDE_API_KEY", true);
+export const posthogApiKey = getEnv("POSTHOG_KEY", true);
+export const posthogHost = getEnv("POSTHOG_HOST", true);
 
 if (!ok) throw new Error("Environment misconfigured");
 console.log("");
diff --git a/app/helpers/metrics.ts b/app/helpers/metrics.ts
diff --git a/index.dev.js b/index.dev.js
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
diff --git a/vite.config.ts b/vite.config.ts
