Skip to content

Commit 90e2e04

Browse files
rajbosrajbos
authored
Merge pull request #122 from RHSplinter/update-csv-format
Update CSV parser to accept the new CSV format
2 parents 77931a8 + dd8fd2b commit 90e2e04

File tree

3 files changed

+108
-86
lines changed

3 files changed

+108
-86
lines changed

‎README.md‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ A single-page application that visualizes GitHub Copilot premium request usage d
77
Need to analyze the premium requests CSV?
88
I created a SPA with Spark and Coding Agent to display an overview of the Premium Requests CSV that you can currently download (no API yet 😓), so share it where needed!
99

10-
Hosted on GitHub Pages: [GitHub Copilot Premium Requests Usage Analyzer](https://devops-actions.github.io/github-copilot-premium-reqs-usage/)
10+
Hosted on GitHub Pages: [GitHub Copilot Premium Requests Usage Analyzer](https://xebia.github.io/github-copilot-premium-reqs-usage/)
1111

1212
Upload the CSV from the enterprise export (Billing and Licenses → Usage → Export dropdown right top)
1313

@@ -53,15 +53,25 @@ Result:
5353

5454
4. Open http://localhost:5000 in your browser
5555

56+
5657
## CSV Format
5758

58-
The application expects a CSV export from GitHub Copilot premium requests with the following format:
59+
The application accepts either of the following CSV export formats from GitHub Copilot premium requests:
5960

61+
**Original format (until October):**
6062
```
6163
"Timestamp","User","Model","Requests Used","Exceeds Monthly Quota","Total Monthly Quota"
6264
"2025-06-11T05:13:27.8766440Z","UserName","gpt-4.1-2025-04-14","1","False","Unlimited"
6365
```
6466

67+
**New format (starting from October):**
68+
```
69+
"date","username","model","quantity","exceeds_quota","total_monthly_quota"
70+
"2025-06-11","UserName","GPT-5","1","False","300"
71+
```
72+
73+
> Column order does not matter and extra columns are ignored. Both formats are supported for backward compatibility.
74+
6575
## Deployment
6676

6777
### GitHub Pages

‎src/lib/utils.ts‎

Lines changed: 96 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -26,98 +26,120 @@ export function parseCSV(csv: string): CopilotUsageData[] {
2626
if (lines.length < 2) {
2727
throw new Error('CSV must contain a header row and at least one data row');
2828
}
29-
30-
// Validate header row
31-
const headerLine = lines[0];
32-
const expectedHeaders = ['Timestamp', 'User', 'Model', 'Requests Used', 'Exceeds Monthly Quota', 'Total Monthly Quota'];
33-
34-
// Parse header row to check for expected columns
35-
// First, trim any trailing whitespace from the header line
36-
const trimmedHeaderLine = headerLine.trim();
37-
const headerMatches = trimmedHeaderLine.match(/("([^"]*)"|([^,]*))(,|$)/g);
38-
if (!headerMatches || headerMatches.length < 6) {
39-
throw new Error('CSV header must contain at least 6 columns');
29+
30+
// Parse header row and build a mapping from expected field to column index (case-insensitive)
31+
const headerLine = lines[0].trim();
32+
const headerMatches = headerLine.match(/("([^"]*)"|([^,]*))(,|$)/g);
33+
if (!headerMatches) {
34+
throw new Error('CSV header could not be parsed');
4035
}
41-
4236
const headers = headerMatches.map(m => {
43-
// Remove trailing comma if present
4437
let processed = m.endsWith(',') ? m.slice(0, -1) : m;
45-
// Remove surrounding quotes if present
4638
processed = processed.replace(/^"(.*)"$/, '$1');
47-
return processed;
48-
}).filter(h => h.trim() !== '').map(h => h.trim()); // Filter empty strings and trim whitespace
49-
50-
// Check if all expected headers are present (case-insensitive exact match)
51-
const missingHeaders = expectedHeaders.filter(expected =>
52-
!headers.some(header => header.toLowerCase() === expected.toLowerCase())
53-
);
54-
55-
// Log detailed header information for debugging
56-
if (missingHeaders.length > 0) {
57-
console.log('CSV Header validation failed:');
58-
console.log('Expected headers:', expectedHeaders);
59-
console.log('Found headers:', headers);
60-
console.log('Missing headers:', missingHeaders);
61-
headers.forEach((header, i) => {
62-
const expectedHeader = expectedHeaders[i];
63-
if (expectedHeader) {
64-
const matches = header.toLowerCase() === expectedHeader.toLowerCase();
65-
console.log(` Column ${i + 1}: "${header}" ${matches ? '✅' : '❌'} (expected: "${expectedHeader}")`);
66-
} else {
67-
console.log(` Column ${i + 1}: "${header}" (extra column)`);
68-
}
69-
});
70-
}
71-
72-
if (missingHeaders.length > 0) {
73-
throw new Error(`CSV is missing required columns: ${missingHeaders.join(', ')}. Expected columns: ${expectedHeaders.join(', ')}`);
39+
return processed.trim();
40+
});
41+
42+
// Map new CSV field names to expected fields (case-insensitive)
43+
const FIELD_MAP: Record<string, string> = {
44+
'date': 'timestamp',
45+
'username': 'user',
46+
'quantity': 'requestsUsed',
47+
'exceeds_quota': 'exceedsQuota',
48+
'total_monthly_quota': 'totalMonthlyQuota',
49+
// Backward compatibility (old headers)
50+
'timestamp': 'timestamp',
51+
'user': 'user',
52+
'model': 'model',
53+
'requests used': 'requestsUsed',
54+
'exceeds monthly quota': 'exceedsQuota',
55+
'total monthly quota': 'totalMonthlyQuota',
56+
};
57+
58+
// For error messages, map internal field names to original header names
59+
const INTERNAL_TO_HEADER: Record<string, string> = {
60+
'timestamp': 'Timestamp',
61+
'user': 'User',
62+
'model': 'Model',
63+
'requestsUsed': 'Requests Used',
64+
'exceedsQuota': 'Exceeds Monthly Quota',
65+
'totalMonthlyQuota': 'Total Monthly Quota',
66+
};
67+
68+
// Build a mapping from expected field to column index (case-insensitive)
69+
const fieldToIndex: Partial<Record<keyof CopilotUsageData, number>> = {};
70+
headers.forEach((header, idx) => {
71+
const mapped = FIELD_MAP[header.toLowerCase()];
72+
if (mapped) {
73+
fieldToIndex[mapped as keyof CopilotUsageData] = idx;
74+
}
75+
});
76+
77+
// Ensure all required fields are present
78+
const requiredFields: Array<keyof CopilotUsageData> = [
79+
'timestamp', 'user', 'model', 'requestsUsed', 'exceedsQuota', 'totalMonthlyQuota'
80+
];
81+
const missingFields = requiredFields.filter(f => fieldToIndex[f] === undefined);
82+
if (missingFields.length > 0) {
83+
// If all columns are missing, check for too few columns
84+
if (headers.length < 6) {
85+
throw new Error('CSV header must contain at least 6 columns');
86+
}
87+
// Compose error message with original header names
88+
const missingHeaderNames = missingFields.map(f => INTERNAL_TO_HEADER[f]);
89+
throw new Error(`CSV is missing required columns: ${missingHeaderNames.join(', ')}. Expected columns: ${Object.values(INTERNAL_TO_HEADER).join(', ')}`);
7490
}
75-
76-
// Skip the header row and process data rows
91+
92+
// Parse data rows
7793
return lines.slice(1).map((line, index) => {
78-
// Handle quoted CSV properly - trim any trailing whitespace first
7994
const trimmedLine = line.trim();
80-
const matches = trimmedLine.match(/("([^"]*)"|([^,]*))(,|$)/g);
81-
82-
if (!matches || matches.length < 6) {
83-
throw new Error(`Invalid CSV row format at line ${index + 2}: expected 6 columns, got ${matches ? matches.length : 0}`);
95+
if (!trimmedLine) return null;
96+
const matches = trimmedLine.match(/("([^"]*)"|([^,]*))(,|$)/g);
97+
if (!matches) {
98+
throw new Error(`Invalid CSV row format at line ${index + 2}`);
8499
}
85-
86-
const values = matches.map(m => {
87-
// Remove trailing comma if present
88-
let processed = m.endsWith(',') ? m.slice(0, -1) : m;
89-
// Remove surrounding quotes if present
90-
processed = processed.replace(/^"(.*)"$/, '$1');
91-
return processed;
92-
}).filter(v => v.trim() !== ''); // Filter out empty values
93-
94-
// Validate timestamp
95-
const timestamp = new Date(values[0]);
100+
// Pad matches to header length (in case of trailing commas)
101+
while (matches.length < headers.length) matches.push('');
102+
103+
// Extract values by mapped index
104+
const getValue = (field: keyof CopilotUsageData) => {
105+
const idx = fieldToIndex[field]!;
106+
let val = matches[idx] || '';
107+
val = val.endsWith(',') ? val.slice(0, -1) : val;
108+
val = val.replace(/^"(.*)"$/, '$1');
109+
return val.trim();
110+
};
111+
112+
// Validate and parse fields
113+
const timestampStr = getValue('timestamp');
114+
const timestamp = new Date(timestampStr);
96115
if (isNaN(timestamp.getTime())) {
97-
throw new Error(`Invalid timestamp format at line ${index + 2}: "${values[0]}"`);
116+
throw new Error(`Invalid timestamp format at line ${index + 2}: "${timestampStr}"`);
98117
}
99-
100-
// Validate requests used
101-
const requestsUsed = parseFloat(values[3]);
118+
119+
const user = getValue('user');
120+
const model = getValue('model');
121+
const requestsUsedStr = getValue('requestsUsed');
122+
const requestsUsed = parseFloat(requestsUsedStr);
102123
if (isNaN(requestsUsed)) {
103-
throw new Error(`Invalid requests used value at line ${index + 2}: "${values[3]}" must be a number`);
124+
throw new Error(`Invalid requests used value at line ${index + 2}: "${requestsUsedStr}" must be a number`);
104125
}
105-
106-
// Validate exceeds quota
107-
const exceedsQuotaValue = values[4].toLowerCase();
126+
127+
const exceedsQuotaValue = getValue('exceedsQuota').toLowerCase();
108128
if (exceedsQuotaValue !== 'true' && exceedsQuotaValue !== 'false') {
109-
throw new Error(`Invalid exceeds quota value at line ${index + 2}: "${values[4]}" must be "true" or "false"`);
129+
throw new Error(`Invalid exceeds quota value at line ${index + 2}: "${getValue('exceedsQuota')}" must be "true" or "false"`);
110130
}
111-
131+
132+
const totalMonthlyQuota = getValue('totalMonthlyQuota');
133+
112134
return {
113135
timestamp,
114-
user: values[1],
115-
model: values[2],
136+
user,
137+
model,
116138
requestsUsed,
117-
exceedsQuota: exceedsQuotaValue === "true",
118-
totalMonthlyQuota: values[5],
139+
exceedsQuota: exceedsQuotaValue === 'true',
140+
totalMonthlyQuota,
119141
};
120-
});
142+
}).filter(Boolean) as CopilotUsageData[];
121143
}
122144

123145
export interface ModelUsageSummary {

‎src/test/csv-header-validation.test.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -50,16 +50,6 @@ describe('CSV Header Validation', () => {
5050
)
5151
})
5252

53-
it('should validate that headers are present regardless of order (but data parsing expects fixed order)', () => {
54-
// Note: The current implementation validates headers are present but still reads data by fixed positions
55-
// This test verifies that header validation works, but the data parsing might fail if order is wrong
56-
const csvDifferentOrder = `"User","Timestamp","Model","Total Monthly Quota","Requests Used","Exceeds Monthly Quota"
57-
user1,2024-01-01T00:00:00Z,gpt-4,100,1.5,false`
58-
59-
// This should not throw a header validation error (headers are all present)
60-
// But it may throw a data parsing error since data is in wrong order
61-
expect(() => parseCSV(csvDifferentOrder)).toThrow('Invalid timestamp format at line 2')
62-
})
6353

6454
it('should handle CSV with mixed case headers and validate correctly', () => {
6555
const csvMixedCase = `"TIMESTAMP","user","Model","REQUESTS USED","exceeds monthly quota","Total Monthly Quota"

0 commit comments

Comments
 (0)