Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/codex/src/commands/daily.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export const dailyCommand = define({
process.exit(1);
}

const { events, missingDirectories } = await loadTokenUsageEvents();
const { events, missingDirectories } = await loadTokenUsageEvents({ since, until });

for (const missing of missingDirectories) {
logger.warn(`Codex session directory not found: ${missing}`);
Expand Down
2 changes: 1 addition & 1 deletion apps/codex/src/commands/monthly.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export const monthlyCommand = define({
process.exit(1);
}

const { events, missingDirectories } = await loadTokenUsageEvents();
const { events, missingDirectories } = await loadTokenUsageEvents({ since, until });

for (const missing of missingDirectories) {
logger.warn(`Codex session directory not found: ${missing}`);
Expand Down
2 changes: 1 addition & 1 deletion apps/codex/src/commands/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export const sessionCommand = define({
process.exit(1);
}

const { events, missingDirectories } = await loadTokenUsageEvents();
const { events, missingDirectories } = await loadTokenUsageEvents({ since, until });

for (const missing of missingDirectories) {
logger.warn(`Codex session directory not found: ${missing}`);
Expand Down
121 changes: 116 additions & 5 deletions apps/codex/src/data-loader.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { TokenUsageDelta, TokenUsageEvent } from './_types.ts';
import { readFile, stat } from 'node:fs/promises';
import { readdir, readFile, stat } from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';
import { Result } from '@praha/byethrow';
Expand Down Expand Up @@ -177,14 +177,84 @@ function asNonEmptyString(value: unknown): string | undefined {

export type LoadOptions = {
sessionDirs?: string[];
since?: string;
until?: string;
};

/**
* List session JSONL files, skipping date directories outside [since, until].
*
* Codex stores sessions as `YYYY/MM/DD/*.jsonl`. When a date range is provided
* we enumerate the directory tree and prune entire year/month/day subtrees that
* cannot contain matching sessions, avoiding the cost of a full recursive glob
* over potentially large historical archives.
*/
async function listSessionFiles(
sessionsDir: string,
since: string | undefined,
until: string | undefined,
): Promise<string[]> {
if (since == null && until == null) {
return glob(SESSION_GLOB, { cwd: sessionsDir, absolute: true });
}

const sinceKey = since?.replaceAll('-', '');
const untilKey = until?.replaceAll('-', '');

Comment on lines +192 to +203
Copy link

Copilot AI Mar 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

listSessionFiles() introduces new range-pruning behavior but there are no tests exercising it (e.g., that it returns only files within [since, until], and that it still includes root-level .jsonl files if they exist). Since this file already contains vitest coverage for loadTokenUsageEvents, adding a focused test case that sets since/until and uses a dated YYYY/MM/DD fixture directory structure would help prevent regressions in the pruning logic.

Copilot uses AI. Check for mistakes.
const tryReaddir = async (dir: string): Promise<string[]> => {
const result = await Result.try({
try: readdir(dir),
catch: (error) => error,
});
return Result.isFailure(result) ? [] : result.value;
};

const files: string[] = [];

for (const year of (await tryReaddir(sessionsDir)).filter((e) => /^\d{4}$/.test(e))) {
if (sinceKey != null && `${year}1231` < sinceKey) {
continue;
}
if (untilKey != null && `${year}0101` > untilKey) {
Comment on lines +214 to +218
Copy link

Copilot AI Mar 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When since/until is set, this implementation only traverses YYYY/MM/DD subdirectories and will ignore any .jsonl files stored directly under sessionsDir (or under non-date subdirs). The existing vitest fixtures in this file create sessions/project-1.jsonl and sessions/legacy.jsonl at the root, so running loadTokenUsageEvents({ since, until }) against a flat layout would incorrectly return no events. Consider also including glob('*.jsonl', { cwd: sessionsDir, absolute: true }) (and/or a small non-recursive fallback) alongside the pruned traversal so date filters don’t break non-date session layouts.

Copilot uses AI. Check for mistakes.
continue;
}

const yearDir = path.join(sessionsDir, year);
for (const month of (await tryReaddir(yearDir)).filter((e) => /^\d{2}$/.test(e))) {
if (sinceKey != null && `${year + month}31` < sinceKey) {
continue;
}
if (untilKey != null && `${year + month}01` > untilKey) {
continue;
}

const monthDir = path.join(yearDir, month);
for (const day of (await tryReaddir(monthDir)).filter((e) => /^\d{2}$/.test(e))) {
const dateKey = year + month + day;
if (sinceKey != null && dateKey < sinceKey) {
continue;
}
if (untilKey != null && dateKey > untilKey) {
continue;
}

const dayDir = path.join(monthDir, day);
const dayFiles = await glob('*.jsonl', { cwd: dayDir, absolute: true }).catch(() => []);
files.push(...dayFiles);
}
}
}

return files;
}
Comment on lines +192 to +249
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Keep legacy sessions/*.jsonl files in filtered runs.

The filtered branch only walks YYYY/MM/DD, so any flat session files under sessionsDir stop participating as soon as since or until is set. This loader still supports that layout elsewhere in the file, so date-filtered runs can undercount or return no data on existing stores.

Suggested compatibility fix
 async function listSessionFiles(
 	sessionsDir: string,
 	since: string | undefined,
 	until: string | undefined,
 ): Promise<string[]> {
 	if (since == null && until == null) {
 		return glob(SESSION_GLOB, { cwd: sessionsDir, absolute: true });
 	}
 
+	// Preserve support for legacy flat layouts when filters are enabled.
+	const rootFiles = await glob('*.jsonl', { cwd: sessionsDir, absolute: true }).catch(() => []);
+
 	const sinceKey = since?.replaceAll('-', '');
 	const untilKey = until?.replaceAll('-', '');
 
-	const files: string[] = [];
+	const files: string[] = [...rootFiles];
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@apps/codex/src/data-loader.ts` around lines 192 - 249, The current
listSessionFiles function ignores flat session files in sessionsDir when
since/until are set; restore compatibility by explicitly globbing SESSION_GLOB
at the root even in the filtered branch and applying the same date filtering
logic: inside listSessionFiles (before or alongside the year/month/day
traversal) call glob(SESSION_GLOB, { cwd: sessionsDir, absolute: true }) to get
rootFiles, derive a date key from each root file's basename (e.g., strip
extension and non-digits to produce YYYYMMDD or YYYYMMDD-like key), compare that
dateKey against sinceKey/untilKey the same way you do for dateKey in the day
loop, and push matching files into files; use existing helpers (tryReaddir) and
constants (SESSION_GLOB) so behavior is consistent with the nested traversal.


export type LoadResult = {
events: TokenUsageEvent[];
missingDirectories: string[];
};

export async function loadTokenUsageEvents(options: LoadOptions = {}): Promise<LoadResult> {
const { since, until } = options;
const providedDirs =
options.sessionDirs != null && options.sessionDirs.length > 0
? options.sessionDirs.map((dir) => path.resolve(dir))
Expand Down Expand Up @@ -216,10 +286,7 @@ export async function loadTokenUsageEvents(options: LoadOptions = {}): Promise<L
continue;
}

const files = await glob(SESSION_GLOB, {
cwd: directoryPath,
absolute: true,
});
const files = await listSessionFiles(directoryPath, since, until);

for (const file of files) {
const relativeSessionPath = path.relative(directoryPath, file);
Expand Down Expand Up @@ -453,6 +520,50 @@ if (import.meta.vitest != null) {
expect(second.cachedInputTokens).toBe(100);
});

it('skips date directories outside the since/until range', async () => {
const makeEvent = (timestamp: string, input_tokens: number) =>
JSON.stringify({
timestamp,
type: 'event_msg',
payload: {
type: 'token_count',
info: {
last_token_usage: {
input_tokens,
cached_input_tokens: 0,
output_tokens: 100,
reasoning_output_tokens: 0,
total_tokens: input_tokens + 100,
},
model: 'gpt-5',
},
},
});

// Fixture mirrors real Codex layout: YYYY/MM/DD/*.jsonl
await using fixture = await createFixture({
'2025': {
'12': {
'31': { 'old.jsonl': makeEvent('2025-12-31T12:00:00.000Z', 999) },
},
},
'2026': {
'03': {
'01': { 'new.jsonl': makeEvent('2026-03-01T12:00:00.000Z', 1_000) },
},
},
});

// With since=2026-03-01 the 2025/12/31 file should be skipped entirely.
const { events } = await loadTokenUsageEvents({
sessionDirs: [fixture.getPath('.')],
since: '2026-03-01',
});

expect(events).toHaveLength(1);
expect(events[0]!.inputTokens).toBe(1_000);
});

it('falls back to legacy model when metadata is missing entirely', async () => {
await using fixture = await createFixture({
sessions: {
Expand Down