Skip to content

Commit 1d3b3a4

Browse files
jischeindevin-ai-integration[bot]chdeskur
authored
feat(analytics): add success / failure count to api explorer table (#5969)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Catherine Deskur <[email protected]> Co-authored-by: Catherine Deskur <[email protected]>
1 parent b1b855c commit 1d3b3a4

File tree

19 files changed

+861
-77
lines changed

19 files changed

+861
-77
lines changed

‎packages/fern-dashboard/scripts/list-production-domains.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ Options:
6868
const source = values["force-fdr"] ? "FDR (forced)" : hasKV ? "KV store" : "FDR";
6969
console.log(`Using ${source} to fetch domains...\n`);
7070

71-
const getAllProductionDomainsModule = await import("../src/app/services/analyticsCron/getAllProductionDomains");
71+
const getAllProductionDomainsModule = await import("../src/app/services/analytics/cron/getAllProductionDomains");
7272

7373
try {
7474
if (values["include-previews"]) {

‎packages/fern-dashboard/scripts/run-test-with-env.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
set -a
3+
source .env.local
4+
set +a
5+
npx tsx "$@"

‎packages/fern-dashboard/scripts/test-api-explorer-redshift.ts‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
import { getRedshiftPool } from "../src/app/services/analytics/redshift-client";
2+
3+
async function testAPIExplorerRedshift() {
4+
const domain = process.argv[2] || "buildwithfern.com";
5+
const pool = getRedshiftPool();
6+
7+
const endDate = new Date();
8+
const startDate = new Date();
9+
startDate.setDate(startDate.getDate() - 7);
10+
11+
console.log("Testing API Explorer Redshift queries for:", domain);
12+
console.log("Date range:", startDate.toISOString(), "to", endDate.toISOString());
13+
console.log("\n");
14+
15+
try {
16+
// Check event counts
17+
console.log("=== Event Counts ===");
18+
19+
const sentCountQuery = `
20+
SELECT COUNT(*) as count
21+
FROM posthog.events
22+
WHERE
23+
event = 'api_playground_request_sent'
24+
AND (
25+
properties."$host"::VARCHAR = $1
26+
OR properties."$host"::VARCHAR = $2
27+
)
28+
AND timestamp >= $3
29+
AND timestamp < $4
30+
`;
31+
32+
const receivedCountQuery = `
33+
SELECT COUNT(*) as count
34+
FROM posthog.events
35+
WHERE
36+
event = 'api_playground_request_received'
37+
AND (
38+
properties."$host"::VARCHAR = $1
39+
OR properties."$host"::VARCHAR = $2
40+
)
41+
AND timestamp >= $3
42+
AND timestamp < $4
43+
`;
44+
45+
const [sentCountResult, receivedCountResult] = await Promise.all([
46+
pool.query(sentCountQuery, [domain, `www.${domain}`, startDate.toISOString(), endDate.toISOString()]),
47+
pool.query(receivedCountQuery, [domain, `www.${domain}`, startDate.toISOString(), endDate.toISOString()])
48+
]);
49+
50+
const sentCount = parseInt(sentCountResult.rows[0].count);
51+
const receivedCount = parseInt(receivedCountResult.rows[0].count);
52+
console.log(`api_playground_request_sent: ${sentCount} events`);
53+
console.log(`api_playground_request_received: ${receivedCount} events`);
54+
console.log(
55+
`Missing received events: ${sentCount - receivedCount} (${(((sentCount - receivedCount) / sentCount) * 100).toFixed(1)}%)`
56+
);
57+
58+
// Check response status distribution in received events
59+
console.log("\n=== Response Status Distribution ===");
60+
const statusQuery = `
61+
SELECT
62+
JSON_SERIALIZE(properties) as props_json
63+
FROM posthog.events
64+
WHERE
65+
event = 'api_playground_request_received'
66+
AND (
67+
properties."$host"::VARCHAR = $1
68+
OR properties."$host"::VARCHAR = $2
69+
)
70+
AND timestamp >= $3
71+
AND timestamp < $4
72+
`;
73+
74+
const statusResult = await pool.query(statusQuery, [
75+
domain,
76+
`www.${domain}`,
77+
startDate.toISOString(),
78+
endDate.toISOString()
79+
]);
80+
81+
const statusCounts: Record<string, number> = {};
82+
for (const row of statusResult.rows) {
83+
const props = JSON.parse(row.props_json);
84+
const status = props.responseStatus;
85+
const statusRange =
86+
status >= 500 ? "5xx" : status >= 400 ? "4xx" : status >= 300 ? "3xx" : status >= 200 ? "2xx" : "other";
87+
statusCounts[statusRange] = (statusCounts[statusRange] || 0) + 1;
88+
}
89+
90+
console.log("Status code distribution:");
91+
Object.entries(statusCounts).forEach(([range, count]) => {
92+
console.log(` ${range}: ${count} (${((count / receivedCount) * 100).toFixed(1)}%)`);
93+
});
94+
95+
// Run actual aggregation
96+
console.log("\n=== Running Aggregation ===");
97+
98+
// Common WHERE clause (identical for both queries)
99+
const commonWhereClause = `
100+
(
101+
properties."$host"::VARCHAR = $1
102+
OR properties."$host"::VARCHAR = $2
103+
)
104+
AND timestamp >= $3
105+
AND timestamp < $4
106+
`;
107+
108+
const sentQuery = `
109+
SELECT JSON_SERIALIZE(properties) as props_json
110+
FROM posthog.events
111+
WHERE
112+
event = 'api_playground_request_sent'
113+
AND ${commonWhereClause}
114+
`;
115+
116+
const receivedQuery = `
117+
SELECT JSON_SERIALIZE(properties) as props_json
118+
FROM posthog.events
119+
WHERE
120+
event = 'api_playground_request_received'
121+
AND ${commonWhereClause}
122+
`;
123+
124+
const [sentResult, receivedResult] = await Promise.all([
125+
pool.query(sentQuery, [domain, `www.${domain}`, startDate.toISOString(), endDate.toISOString()]),
126+
pool.query(receivedQuery, [domain, `www.${domain}`, startDate.toISOString(), endDate.toISOString()])
127+
]);
128+
129+
// Build docsRoute mapping
130+
const docsRouteToEndpoint = new Map<string, { endpointRoute: string; endpointName: string; method: string }>();
131+
for (const row of sentResult.rows) {
132+
const props = JSON.parse(row.props_json);
133+
const docsRoute = props.docsRoute || "";
134+
const endpointRoute = props.endpointRoute || "";
135+
const endpointName = props.endpointName || "";
136+
const method = props.method || "";
137+
if (docsRoute) {
138+
docsRouteToEndpoint.set(docsRoute, { endpointRoute, endpointName, method });
139+
}
140+
}
141+
142+
// Aggregate by endpointRoute
143+
const counts = new Map<
144+
string,
145+
{ method: string; endpoint: string; name: string; count: number; numSuccesses: number; numFailures: number }
146+
>();
147+
148+
for (const row of sentResult.rows) {
149+
const props = JSON.parse(row.props_json);
150+
const method = props.method || "";
151+
const endpointRoute = props.endpointRoute || "";
152+
const endpointName = props.endpointName || "";
153+
154+
const key = `${method}|${endpointRoute}|${endpointName}`;
155+
const existing = counts.get(key) || {
156+
method,
157+
endpoint: endpointRoute,
158+
name: endpointName,
159+
count: 0,
160+
numSuccesses: 0,
161+
numFailures: 0
162+
};
163+
existing.count++;
164+
counts.set(key, existing);
165+
}
166+
167+
// Add status codes from received events
168+
for (const row of receivedResult.rows) {
169+
const props = JSON.parse(row.props_json);
170+
const docsRoute = props.docsRoute || "";
171+
const responseStatus = props.responseStatus;
172+
173+
const mappedEndpoint = docsRouteToEndpoint.get(docsRoute);
174+
if (!mappedEndpoint) {
175+
continue;
176+
}
177+
178+
const key = `${mappedEndpoint.method}|${mappedEndpoint.endpointRoute}|${mappedEndpoint.endpointName}`;
179+
const existing = counts.get(key);
180+
181+
if (existing) {
182+
if (responseStatus >= 200 && responseStatus < 300) {
183+
existing.numSuccesses++;
184+
} else if (responseStatus >= 400) {
185+
existing.numFailures++;
186+
}
187+
}
188+
}
189+
190+
// Show top endpoints
191+
console.log("\n=== Top 10 Endpoints ===");
192+
const sorted = Array.from(counts.values())
193+
.sort((a, b) => b.count - a.count)
194+
.slice(0, 10);
195+
196+
sorted.forEach((data, idx) => {
197+
const unaccounted = data.count - data.numSuccesses - data.numFailures;
198+
const unaccountedPct = ((unaccounted / data.count) * 100).toFixed(1);
199+
console.log(`\n${idx + 1}. ${data.method} ${data.endpoint || data.name}`);
200+
console.log(` Total: ${data.count}`);
201+
console.log(` Successes (2xx): ${data.numSuccesses}`);
202+
console.log(` Failures (4xx/5xx): ${data.numFailures}`);
203+
console.log(` Unaccounted (missing/3xx): ${unaccounted} (${unaccountedPct}%)`);
204+
});
205+
206+
// Summary
207+
console.log("\n=== SUMMARY ===");
208+
const totalRequests = Array.from(counts.values()).reduce((sum, d) => sum + d.count, 0);
209+
const totalSuccesses = Array.from(counts.values()).reduce((sum, d) => sum + d.numSuccesses, 0);
210+
const totalFailures = Array.from(counts.values()).reduce((sum, d) => sum + d.numFailures, 0);
211+
const totalUnaccounted = totalRequests - totalSuccesses - totalFailures;
212+
213+
console.log(`Total requests (sent): ${totalRequests}`);
214+
console.log(
215+
`Total successes (2xx): ${totalSuccesses} (${((totalSuccesses / totalRequests) * 100).toFixed(1)}%)`
216+
);
217+
console.log(
218+
`Total failures (4xx/5xx): ${totalFailures} (${((totalFailures / totalRequests) * 100).toFixed(1)}%)`
219+
);
220+
console.log(
221+
`Total unaccounted: ${totalUnaccounted} (${((totalUnaccounted / totalRequests) * 100).toFixed(1)}%)`
222+
);
223+
console.log(`\nPossible reasons for unaccounted requests:`);
224+
console.log(` - Request timeout (no response received)`);
225+
console.log(` - Network error (no response received)`);
226+
console.log(` - 3xx redirects (not counted as success or failure)`);
227+
console.log(` - 1xx informational responses`);
228+
console.log(` - Response not logged to PostHog`);
229+
230+
await pool.end();
231+
console.log("\n✓ Test completed!");
232+
process.exit(0);
233+
} catch (error) {
234+
console.error("❌ Error:", error);
235+
await pool.end();
236+
process.exit(1);
237+
}
238+
}
239+
240+
testAPIExplorerRedshift();

‎packages/fern-dashboard/src/app/actions/getWebAnalytics.ts‎

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ import { z } from "zod";
44

55
import type { AnalyticsField, AnalyticsSortDir } from "@/components/web-analytics/constants";
66

7-
import { insertAnalyticsForSite } from "../services/analyticsCron/insert";
8-
import type { DateRangePeriod } from "../services/analyticsCron/types";
7+
import { insertAnalyticsForSite } from "../services/analytics/cron/insert";
8+
import type { DateRangePeriod } from "../services/analytics/cron/types";
99
import { getCurrentSessionOrThrow } from "../services/auth0/getCurrentSession";
1010
import { getAnalyticsService } from "../services/posthog";
1111
import { type CachedAnalytics, getCachedAnalytics } from "../services/posthog/cache";
@@ -65,7 +65,7 @@ const GetWebAnalyticsSchema = z.object({
6565

6666
const TableRequestSchema = GetWebAnalyticsSchema.extend({
6767
limit: z.number().int().min(1).max(100).optional(),
68-
orderBy: z.enum(["visitors", "views"]).optional(),
68+
orderBy: z.enum(["visitors", "views", "count", "numSuccesses", "numFailures"]).optional(),
6969
order: z.enum(["asc", "desc"]).optional()
7070
});
7171

@@ -114,6 +114,8 @@ export interface AllAnalyticsResponse {
114114
endpoint: string;
115115
name: string;
116116
count: number;
117+
numSuccesses: number;
118+
numFailures: number;
117119
}[];
118120
llmBotTraffic: { provider: string; count: number }[];
119121
pages404: { path: string; count: number }[];
@@ -293,7 +295,9 @@ export async function getAllAnalytics(request: GetWebAnalyticsRequest): Promise<
293295
method: a.method,
294296
endpoint: a.endpoint,
295297
name: a.name,
296-
count: a.count
298+
count: a.count,
299+
numSuccesses: a.numSuccesses || 0,
300+
numFailures: a.numFailures || 0
297301
})),
298302
llmBotTraffic: supabaseCache.topLlmBotTraffic,
299303
pages404: [],
@@ -383,7 +387,9 @@ export async function getAllAnalytics(request: GetWebAnalyticsRequest): Promise<
383387
method: a.method,
384388
endpoint: a.endpoint,
385389
name: a.name,
386-
count: a.requests
390+
count: a.requests,
391+
numSuccesses: a.numSuccesses,
392+
numFailures: a.numFailures
387393
})),
388394
llmBotTraffic: llmBotTraffic.map((b) => ({ provider: b.provider, count: b.requests })),
389395
pages404: [], // Not implemented in Redshift
@@ -461,7 +467,7 @@ export async function getTopPages(
461467
const validated = TableRequestSchema.parse(request);
462468
const dateRange = validated.dateRange || DEFAULT_DATE_RANGE;
463469
const limit = validated.limit || 10;
464-
const orderBy = validated.orderBy || "views";
470+
const orderBy = validated.orderBy === "visitors" || validated.orderBy === "views" ? validated.orderBy : "views";
465471
const order = validated.order || "desc";
466472

467473
const handler = getHandler(validated.docsUrl, dateRange, validated.selectedDomain);
@@ -497,7 +503,7 @@ export async function getTopCountries(request: TableRequest): Promise<{
497503
const validated = TableRequestSchema.parse(request);
498504
const dateRange = validated.dateRange || DEFAULT_DATE_RANGE;
499505
const limit = validated.limit || 10;
500-
const orderBy = validated.orderBy || "visitors";
506+
const orderBy = validated.orderBy === "visitors" || validated.orderBy === "views" ? validated.orderBy : "visitors";
501507
const order = validated.order || "desc";
502508

503509
const handler = getHandler(validated.docsUrl, dateRange, validated.selectedDomain);
@@ -580,11 +586,12 @@ export async function getChannels(request: TableRequest): Promise<{
580586

581587
const live = await getLiveAnalytics(validated.docsUrl);
582588
const analytics = live.getAnalytics();
589+
const orderBy = validated.orderBy === "visitors" || validated.orderBy === "views" ? validated.orderBy : "visitors";
583590
const channels = await analytics.getChannels({
584591
dateRange,
585592
includeInternal: validated.includeInternal,
586593
limit: validated.limit || 20,
587-
orderBy: validated.orderBy || "visitors",
594+
orderBy,
588595
order: validated.order || "desc"
589596
});
590597

@@ -609,11 +616,12 @@ export async function getDeviceTypes(request: TableRequest): Promise<{
609616

610617
const live = await getLiveAnalytics(validated.docsUrl);
611618
const analytics = live.getAnalytics();
619+
const orderBy = validated.orderBy === "visitors" || validated.orderBy === "views" ? validated.orderBy : "visitors";
612620
const deviceTypes = await analytics.getDeviceTypes({
613621
dateRange,
614622
includeInternal: validated.includeInternal,
615623
limit: validated.limit || 10,
616-
orderBy: validated.orderBy || "visitors",
624+
orderBy,
617625
order: validated.order || "desc"
618626
});
619627

@@ -638,11 +646,12 @@ export async function getReferringDomains(request: TableRequest): Promise<{
638646

639647
const live = await getLiveAnalytics(validated.docsUrl);
640648
const analytics = live.getAnalytics();
649+
const orderBy = validated.orderBy === "visitors" || validated.orderBy === "views" ? validated.orderBy : "visitors";
641650
const referringDomains = await analytics.getReferringDomains({
642651
dateRange,
643652
includeInternal: validated.includeInternal,
644653
limit: validated.limit || 10,
645-
orderBy: validated.orderBy || "visitors",
654+
orderBy,
646655
order: validated.order || "desc"
647656
});
648657

@@ -673,6 +682,8 @@ export async function getAPIExplorerRequests(request: TableRequest): Promise<{
673682
endpoint: string;
674683
name: string;
675684
count: number;
685+
numSuccesses: number;
686+
numFailures: number;
676687
}[];
677688
}> {
678689
const validated = TableRequestSchema.parse(request);
@@ -689,7 +700,9 @@ export async function getAPIExplorerRequests(request: TableRequest): Promise<{
689700
method: a.method,
690701
endpoint: a.endpoint,
691702
name: a.name,
692-
count: a.count
703+
count: a.count,
704+
numSuccesses: a.numSuccesses || 0,
705+
numFailures: a.numFailures || 0
693706
}))
694707
};
695708
}
@@ -699,7 +712,11 @@ export async function getAPIExplorerRequests(request: TableRequest): Promise<{
699712
const apiExplorerRequests = await analytics.getAPIExplorerRequests({
700713
dateRange,
701714
limit: validated.limit || 20,
702-
order: validated.order || "desc"
715+
order: validated.order || "desc",
716+
orderBy:
717+
validated.orderBy === "count" || validated.orderBy === "numSuccesses" || validated.orderBy === "numFailures"
718+
? validated.orderBy
719+
: "count"
703720
});
704721

705722
return { apiExplorerRequests };

0 commit comments

Comments
 (0)