Skip to content
Merged
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { afterAll, beforeAll, describe, it } from 'vitest';

import { seeders } from '@nangohq/shared';

import { authenticateUser, isSuccess, runServer, shouldBeProtected } from '../../../utils/tests.js';
import { runServer, shouldBeProtected } from '../../../utils/tests.js';

const route = '/api/v1/meta';
let api: Awaited<ReturnType<typeof runServer>>;
Expand All @@ -21,14 +19,4 @@ describe(`GET ${route}`, () => {
const res = await api.fetch(route, { method: 'GET' });
shouldBeProtected(res);
});

it('returns billingUsageSource=clickhouse', async () => {
const { user } = await seeders.seedAccountEnvAndUser();
const session = await authenticateUser(api, user);
// @ts-expect-error type declares `env` but the controller rejects any query param
const res = await api.fetch(route, { method: 'GET', session });
expect(res.res.status).toBe(200);
isSuccess(res.json);
expect(res.json.data.billingUsageSource).toBe('clickhouse');
});
});
3 changes: 1 addition & 2 deletions packages/server/lib/controllers/v1/meta/getMeta.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ export const getMeta = asyncWrapper<GetMeta>(async (req, res) => {
version: NANGO_VERSION,
baseUrl,
debugMode: req.session.debugMode === true,
gettingStartedClosed: sessionUser.getting_started_closed,
billingUsageSource: 'clickhouse'
gettingStartedClosed: sessionUser.getting_started_closed
}
});
});
15 changes: 3 additions & 12 deletions packages/server/lib/controllers/v1/plans/usage/getBillingUsage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,23 +87,15 @@ const querySchema = z
env: z.string(),
from: z.iso.datetime().optional(),
to: z.iso.datetime().optional(),
// Per-request dashboard backend override. Webapp picks it up from
// localStorage('nango.billingUsageSource') and forwards. Honoured
// server-side only when FLAG_ALLOW_OVERRIDE_GETUSAGE_SERVICE is on (dev
// gate). Without the gate, this is ignored and the dashboard stays
// on Orb.
source: z.enum(['clickhouse', 'orb']).optional(),
// Repeated-key array (`?metrics=records&metrics=connections`) —
// scopes the response to just those metrics. Empty / unset → all 7
// (page-load shape). Used by the drilldown UI to fetch just the
// metric the user opened. Honoured only on the CH path; Orb path
// ignores it for now. Preprocess wraps the single-value case
// metric the user opened. Preprocess wraps the single-value case
// (`?metrics=records` → string) into an array so the enum check
// applies uniformly.
metrics: z.preprocess((v) => (typeof v === 'string' ? [v] : v), z.array(z.enum(ALL_METRICS)).nonempty().optional()),
// Per-metric breakdown spec, Express qs parses `breakdown[<metric>]=<dim>`
// into `{ <metric>: <dim>, … }`. Honoured only on the CH path; the
// Orb client ignores it silently for now.
// into `{ <metric>: <dim>, … }`.
breakdown: breakdownSchema,
// Top-N for breakdown. Capped at TOP_N_BREAKDOWN_CAP at the schema level
// so requests exceeding it 400 rather than silently clamping — the SQL
Expand All @@ -116,7 +108,7 @@ const querySchema = z
// rows. The one rejected case is filtering and breaking down by the SAME
// dim (e.g. filter `integration_id:hubspot` + breakdown `integration_id`):
// that produces a single-value "breakdown" — just the filter restated — so
// the refine below 400s it. CH path only.
// the refine below 400s it.
filter: filterSchema
})
.refine(
Expand Down Expand Up @@ -201,7 +193,6 @@ export const getBillingUsage = asyncWrapper<GetBillingUsage>(async (req, res) =>
const usage = await usageTracker.getBillingUsage(plan.orb_subscription_id, account.id, {
granularity: 'day',
...(query.from && query.to ? { timeframe: { start: new Date(query.from), end: new Date(query.to) } } : {}),
...(query.source ? { source: query.source } : {}),
...(query.metrics ? { metrics: query.metrics } : {}),
...(query.breakdown ? { breakdown: query.breakdown } : {}),
...(query.top ? { top: query.top } : {}),
Expand Down
23 changes: 7 additions & 16 deletions packages/types/lib/billing/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,35 +99,26 @@ export interface GetBillingUsageOpts {
group_by?: 'environmentId' | 'environmentName' | 'integrationId' | 'type' | 'functionName' | 'model';
};
/**
* Dev-only escape hatch: pins the request to Orb for parity checks.
* Honoured only when `FLAG_ALLOW_OVERRIDE_GETUSAGE_SERVICE` is enabled;
* ignored everywhere else. Default is ClickHouse.
*/
source?: 'clickhouse' | 'orb';
/**
* Per-metric dimension breakdown spec. Honoured only on the CH path
* Per-metric dimension breakdown spec, applied to CH-backed metrics
* (records / connections via `getDailySumAndBatches`, counters via
* `getDailyCounter`); the Orb client ignores it. Each metric's
* `BillingUsageMetric` gains a `breakdown` array of up to `top + 1`
* series (top-N dimension values + a single 'rest' aggregating the
* long tail). Top defaults to 10 and is clamped server-side to
* the CH cap.
* `getDailyCounter`). Each metric's `BillingUsageMetric` gains a
* `breakdown` array of up to `top + 1` series (top-N dimension
* values + a single 'rest' aggregating the long tail). Top defaults
* to 10 and is clamped server-side to the CH cap.
*/
breakdown?: { [M in UsageMetric]?: BreakdownDimensions[M] | undefined };
top?: number;
/**
* Subset of metrics to populate in the response. When set, only those
* metrics are fanned out (CH path) and returned. Omitted → all 7.
* Ignored on the Orb path.
* metrics are fanned out and returned. Omitted → all 7.
*/
metrics?: UsageMetric[];
/**
* Per-metric row-level filter: scopes that metric's response to rows
* where the given dimension equals the given value. Composes with
* `breakdown[<metric>]` on the same metric when the dimensions differ
* (drill-in: filter to one value, re-break-down by another); controllers
* reject only the same-dimension pairing. CH path only; the Orb client
* ignores it.
* reject only the same-dimension pairing.
*/
filter?: { [M in UsageMetric]?: { dimension: BreakdownDimensions[M]; value: string } | undefined };
}
Expand Down
1 change: 0 additions & 1 deletion packages/types/lib/meta/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ export type GetMeta = Endpoint<{
baseUrl: string;
debugMode: boolean;
gettingStartedClosed: boolean;
billingUsageSource: 'clickhouse' | 'orb';
};
};
}>;
89 changes: 16 additions & 73 deletions packages/usage/lib/usage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import { UsageBillingClient } from './billing.js';
import { UsageCache } from './cache.js';
import { Clickhouse } from './clickhouse/clickhouse.js';
import { AVG_METRICS, COUNTER_METRICS } from './clickhouse/clickhouse.query.js';
import { envs } from './env.js';
import { logger } from './logger.js';
import { usageMetrics } from './metrics.js';

Expand Down Expand Up @@ -323,43 +322,17 @@ export class UsageTracker implements IUsageTracker {
return this.getClickhouse().getTopDimensionValues(params as GetTopDimensionValuesQuery);
}

public async getBillingUsage(subscriptionId: string, accountId: number, opts?: GetBillingUsageOpts): Promise<Result<BillingUsageMetrics>> {
// ClickHouse is the default. `?source=orb` only takes effect when
// FLAG_ALLOW_OVERRIDE_GETUSAGE_SERVICE is on (dev-only parity checks).
const orbOverride = envs.FLAG_ALLOW_OVERRIDE_GETUSAGE_SERVICE && opts?.source === 'orb';
if (!orbOverride) {
if (opts?.granularity === 'day' && opts.timeframe?.start && opts.timeframe?.end) {
return this.getBillingUsageFromClickhouse(accountId, {
timeframe: opts.timeframe,
...(opts.metrics ? { metrics: opts.metrics } : {}),
...(opts.breakdown ? { breakdown: opts.breakdown } : {}),
...(opts.top !== undefined ? { top: opts.top } : {}),
...(opts.filter ? { filter: opts.filter } : {})
});
}
return this.getClickhouse().getCurrentMonthBillingMetrics(accountId, new Date());
}

// Strip CH-only fields so they don't pollute the Orb client's Redis
// cache key. Orb ignores them, but the cache key hashes the full opts
// and would miss on otherwise-identical queries.
const orbOpts: GetBillingUsageOpts | undefined = opts
? {
...(opts.timeframe ? { timeframe: opts.timeframe } : {}),
...(opts.granularity ? { granularity: opts.granularity } : {}),
...(opts.billingMetric ? { billingMetric: opts.billingMetric } : {})
}
: undefined;
const orbResult = await this.billingClient.getUsage(subscriptionId, orbOpts);
if (orbResult.isErr()) {
return Err(orbResult.error);
public async getBillingUsage(_subscriptionId: string, accountId: number, opts?: GetBillingUsageOpts): Promise<Result<BillingUsageMetrics>> {
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
if (opts?.granularity === 'day' && opts.timeframe?.start && opts.timeframe?.end) {
return this.getBillingUsageFromClickhouse(accountId, {
timeframe: opts.timeframe,
...(opts.metrics ? { metrics: opts.metrics } : {}),
...(opts.breakdown ? { breakdown: opts.breakdown } : {}),
...(opts.top !== undefined ? { top: opts.top } : {}),
...(opts.filter ? { filter: opts.filter } : {})
});
}
const orbValue = orbResult.value;
return Ok({
...orbValue,
connections: orbValue.connections ? toCumulativeUsage(orbValue.connections) : undefined,
records: orbValue.records ? toCumulativeUsage(orbValue.records) : undefined
});
return this.getClickhouse().getCurrentMonthBillingMetrics(accountId, new Date());
}

/**
Expand Down Expand Up @@ -656,46 +629,16 @@ const sources: Record<UsageMetric, string> = {
data_transfer: 'billing:subscription:usage'
};

function toCumulativeUsage(periodicUsage: BillingUsageMetric): BillingUsageMetric {
const orderedPeriodicUsage = periodicUsage.usage.sort((a, b) => new Date(a.timeframeStart).getTime() - new Date(b.timeframeStart).getTime());
const cumulativeUsage: BillingUsageMetric['usage'] = [];
let previousQuantity = 0;

for (const usage of orderedPeriodicUsage) {
if (usage?.quantity === undefined) {
cumulativeUsage.push(usage);
continue;
}
const quantity = usage.quantity + previousQuantity;

cumulativeUsage.push({
timeframeStart: usage.timeframeStart,
timeframeEnd: usage.timeframeEnd,
quantity: Math.floor(quantity)
});

previousQuantity = quantity;
}
return {
...periodicUsage,
view_mode: 'cumulative',
total: Math.floor(previousQuantity),
usage: cumulativeUsage
};
}

/**
* CH-path sibling of `toCumulativeUsage`. Turns the per-day `(sum, batches)`
* accumulators returned by `Clickhouse.getDailySumAndBatches` into
* `BillingUsageMetric[]` with `view_mode='cumulative'` — the same wire shape
* the dashboard already consumes for `records` / `connections` from the Orb
* path. One `BillingUsageMetric` per series (no-dim → 1; dim → one per dim
* value with `group: {key, value}`).
* Turns the per-day `(sum, batches)` accumulators returned by
* `Clickhouse.getDailySumAndBatches` into `BillingUsageMetric[]` with
* `view_mode='cumulative'` — the wire shape the dashboard consumes for
* `records` / `connections`. One `BillingUsageMetric` per series (no-dim → 1;
* dim → one per dim value with `group: {key, value}`).
*
* Walks each series in day order, accumulates `running_sum` and
* `running_batches`, and emits `Math.round(running_sum / running_batches)` per
* day. Bypasses `toCumulativeUsage` because the output is already
* cumulative-shaped (running averages, not running sums of deltas).
* day.
*
* Dim-breakdown additivity contract: per-dim series share the same global
* per-day batches (by design of `getDailySumAndBatches`' dim branch), so the
Expand Down
4 changes: 0 additions & 4 deletions packages/utils/lib/environment/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -373,10 +373,6 @@ export const ENVS = z.object({
.number()
.optional()
.default(3600 * 6), // 6 hour
// Dev-only override: allows the `source` query param on `getBillingUsage`
// to pin a request to Orb for parity checks. Default is ClickHouse; when
// OFF (prod default), the `source` param is ignored.
FLAG_ALLOW_OVERRIDE_GETUSAGE_SERVICE: z.stringbool().optional().default(false),

// --- Third parties
// AWS
Expand Down
14 changes: 3 additions & 11 deletions packages/webapp/src/hooks/usePlan.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -101,19 +101,16 @@ export function useApiGetUsage(env: string) {

export const GetBillingUsageQueryKey = ['plans', 'billing-usage'];

export function useApiGetBillingUsage(env: string, timeframe?: { start: string; end: string }, source?: 'clickhouse' | 'orb') {
export function useApiGetBillingUsage(env: string, timeframe?: { start: string; end: string }) {
return useQuery<GetBillingUsage['Success'], APIError>({
enabled: Boolean(env),
queryKey: [...GetBillingUsageQueryKey, timeframe, source],
queryKey: [...GetBillingUsageQueryKey, timeframe],
queryFn: async (): Promise<GetBillingUsage['Success']> => {
const params = new URLSearchParams({ env });
if (timeframe) {
params.append('from', timeframe.start);
params.append('to', timeframe.end);
}
if (source) {
params.append('source', source);
}

const res = await apiFetch(`/api/v1/plans/billing-usage?${params.toString()}`, {
method: 'GET'
Expand Down Expand Up @@ -141,9 +138,7 @@ export function useApiGetBillingUsage(env: string, timeframe?: { start: string;
* - filter + breakdown (different dims) → the breakdown computed within the
* filtered slice, plus a filtered `total` so the headline matches the series.
*
* These are ClickHouse-only features, so the request forces `source=clickhouse`
* (honoured under the dev gate). The caller keeps using `useApiGetBillingUsage`
* for the unfiltered page-load totals.
* The caller keeps using `useApiGetBillingUsage` for the unfiltered page-load totals.
*/
export function useApiGetBillingUsageDetail<M extends UsageMetric>(
env: string,
Expand Down Expand Up @@ -175,9 +170,6 @@ export function useApiGetBillingUsageDetail<M extends UsageMetric>(
params.append('from', timeframe.start);
params.append('to', timeframe.end);
}
// breakdown / filter only exist on the ClickHouse path; force the
// source so it resolves under the dev gate (FLAG_ALLOW_OVERRIDE_GETUSAGE_SERVICE).
params.append('source', 'clickhouse');
params.append('metrics', metric);
if (dimension) {
params.append(`breakdown[${metric}]`, dimension);
Expand Down
8 changes: 1 addition & 7 deletions packages/webapp/src/pages/Team/Billing/components/Usage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import { StyledLink } from '@/components/ui/StyledLink';
import { useEnvironment } from '@/hooks/useEnvironment';
import { useApiGetBillingUsage } from '@/hooks/usePlan';
import { useStore } from '@/store';
import { useBreakdownEnabled } from '../useBreakdownEnabled';
import { useGlobalGroupFilter } from '../useGlobalGroupFilter';
import { UsageChartCard } from './UsageChartCard';

Expand Down Expand Up @@ -39,12 +38,7 @@ export const Usage: React.FC<UsageProps> = ({ selectedMonth }) => {
};
}, [selectedMonth]);

// Pin the whole dashboard to ClickHouse when breakdown is active so headline
// totals match the per-panel breakdowns (which always query ClickHouse).
const breakdownEnabled = useBreakdownEnabled();
const source = breakdownEnabled ? 'clickhouse' : undefined;

const { data: usage, isLoading, error: usageError } = useApiGetBillingUsage(env, timeframe, source);
const { data: usage, isLoading, error: usageError } = useApiGetBillingUsage(env, timeframe);

const { isDivergingFromGlobal, applyToAll } = useGlobalGroupFilter(METRICS);

Expand Down
17 changes: 4 additions & 13 deletions packages/webapp/src/pages/Team/Billing/useBreakdownEnabled.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,8 @@
import { useMeta } from '@/hooks/useMeta';
import { useFeatureFlagsStore } from '@/store/feature-flags';

/**
* Whether the usage breakdown view is shown. Two independent opt-ins:
*
* 1. The account's server-side billing-usage source — `billingUsageSource ===
* 'clickhouse'` from /api/v1/meta, set by the rollout allowlist. This is the
* real signal and takes precedence: once it's on, the dev flag is a no-op.
* 2. The local-storage `usageBreakdown` dev flag, for previewing the feature on
* accounts the rollout hasn't reached yet.
* Whether the usage breakdown view is shown. Always true post-cutover — kept
* as a hook so call sites don't have to change while the always-true
* conditionals downstream get inlined in follow-up cleanup.
*/
export function useBreakdownEnabled(): boolean {
const { data: meta } = useMeta();
const devFlag = useFeatureFlagsStore((s) => s.usageBreakdown);
return meta?.data.billingUsageSource === 'clickhouse' || devFlag;
return true;
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Outdated
}