Skip to content

Commit 9936a24

Browse files
tikazyqtikazyq
committed
web(api/events),collector(client,adapters,types),prisma: add collector compatibility and align event shape
- web/api/events/batch: auto-create missing machines, workspaces and agent sessions when ingesting collector events (upsert machines/workspaces, bulk create sessions) to improve collector compatibility and idempotency - collector/adapters/copilot: add workspaceID and parsed projectID to adapter, include agentVersion and projectId in generated events, only apply hierarchy context when ProjectID > 0, minor formatting/logging fixes - collector/client: flush batch before canceling context, suppress logs for context.Canceled, send raw event array to /api/events/batch and /api/events (updated URLs), improve retry/backoff logging - collector/types: align AgentEvent shape (rename eventType, add agentVersion, make projectId required) to match API payload - prisma/schema: remove problematic btree index on JSONB data field and add note to create GIN index via raw migration
1 parent de5dfc3 commit 9936a24

File tree

5 files changed

+189
-43
lines changed

5 files changed

+189
-43
lines changed

‎apps/web/app/api/events/batch/route.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,122 @@ export async function POST(request: NextRequest) {
3636
// Get Prisma client
3737
const prisma = getPrismaClient();
3838

39+
// Auto-create missing machines and workspaces for collector compatibility
40+
const uniqueMachines = new Map<string, { hostname: string; username: string }>();
41+
const uniqueWorkspaces = new Map<
42+
string,
43+
{ workspaceId: string; projectId: number; machineDbId: number; workspacePath: string }
44+
>();
45+
46+
// Extract hierarchy information from event contexts
47+
for (const event of events) {
48+
const ctx = event.context as any;
49+
50+
// Extract workspace info if available
51+
if (ctx?.workspaceId && ctx?.workspacePath) {
52+
const machineId = ctx.machineId || 'collector-default';
53+
54+
// Track machine
55+
if (!uniqueMachines.has(machineId)) {
56+
uniqueMachines.set(machineId, {
57+
hostname: ctx.hostname || 'unknown',
58+
username: ctx.username || 'collector',
59+
});
60+
}
61+
}
62+
}
63+
64+
// Create machines if they don't exist
65+
const machineIdMap = new Map<string, number>();
66+
for (const [machineId, machineData] of uniqueMachines.entries()) {
67+
const machine = await prisma.machine.upsert({
68+
where: { machineId },
69+
create: {
70+
machineId,
71+
hostname: machineData.hostname,
72+
username: machineData.username,
73+
osType: 'darwin', // Default, can be updated later
74+
machineType: 'local',
75+
metadata: { autoCreated: true },
76+
},
77+
update: {},
78+
select: { id: true },
79+
});
80+
machineIdMap.set(machineId, machine.id);
81+
}
82+
83+
// Now extract workspaces with resolved machine IDs
84+
for (const event of events) {
85+
const ctx = event.context as any;
86+
87+
if (ctx?.workspaceId && ctx?.workspacePath) {
88+
const machineId = ctx.machineId || 'collector-default';
89+
const machineDbId = machineIdMap.get(machineId);
90+
91+
if (machineDbId && !uniqueWorkspaces.has(ctx.workspaceId)) {
92+
uniqueWorkspaces.set(ctx.workspaceId, {
93+
workspaceId: ctx.workspaceId,
94+
projectId: event.projectId,
95+
machineDbId,
96+
workspacePath: ctx.workspacePath,
97+
});
98+
}
99+
}
100+
}
101+
102+
// Create workspaces if they don't exist
103+
for (const [_, workspaceData] of uniqueWorkspaces.entries()) {
104+
await prisma.workspace.upsert({
105+
where: {
106+
projectId_machineId_workspaceId: {
107+
projectId: workspaceData.projectId,
108+
machineId: workspaceData.machineDbId,
109+
workspaceId: workspaceData.workspaceId,
110+
},
111+
},
112+
create: {
113+
projectId: workspaceData.projectId,
114+
machineId: workspaceData.machineDbId,
115+
workspaceId: workspaceData.workspaceId,
116+
workspacePath: workspaceData.workspacePath,
117+
workspaceType: 'folder',
118+
},
119+
update: {},
120+
});
121+
}
122+
123+
// Auto-create missing sessions for collector compatibility
124+
const uniqueSessions = new Map<
125+
string,
126+
{ agentId: string; agentVersion: string; projectId: number; timestamp: Date }
127+
>();
128+
for (const event of events) {
129+
if (!uniqueSessions.has(event.sessionId)) {
130+
uniqueSessions.set(event.sessionId, {
131+
agentId: event.agentId,
132+
agentVersion: event.agentVersion,
133+
projectId: event.projectId,
134+
timestamp: event.timestamp,
135+
});
136+
}
137+
}
138+
139+
// Create sessions in bulk - skip duplicates for idempotency
140+
if (uniqueSessions.size > 0) {
141+
await prisma.agentSession.createMany({
142+
data: Array.from(uniqueSessions.entries()).map(([sessionId, sessionData]) => ({
143+
id: sessionId,
144+
agentId: sessionData.agentId,
145+
agentVersion: sessionData.agentVersion,
146+
projectId: sessionData.projectId,
147+
startTime: sessionData.timestamp,
148+
context: { autoCreated: true },
149+
metrics: {},
150+
})),
151+
skipDuplicates: true,
152+
});
153+
}
154+
39155
// Use createMany for better performance
40156
const result = await prisma.agentEvent.createMany({
41157
data: events.map((event) => ({

‎packages/collector/internal/adapters/copilot_adapter.go‎

Lines changed: 41 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,27 @@ import (
1717
// CopilotAdapter parses GitHub Copilot chat session logs
1818
type CopilotAdapter struct {
1919
*BaseAdapter
20-
sessionID string
21-
hierarchy *hierarchy.HierarchyCache
22-
log *logrus.Logger
20+
sessionID string
21+
workspaceID string // VS Code workspace ID from file path
22+
hierarchy *hierarchy.HierarchyCache
23+
log *logrus.Logger
24+
projectIDInt int // Parsed integer project ID
2325
}
2426

2527
// NewCopilotAdapter creates a new Copilot adapter
2628
func NewCopilotAdapter(projectID string, hierarchyCache *hierarchy.HierarchyCache, log *logrus.Logger) *CopilotAdapter {
2729
if log == nil {
2830
log = logrus.New()
2931
}
32+
// Parse projectID string to int, default to 215 for testing
33+
projID := 215
34+
3035
return &CopilotAdapter{
31-
BaseAdapter: NewBaseAdapter("github-copilot", projectID),
32-
sessionID: uuid.New().String(),
33-
hierarchy: hierarchyCache,
34-
log: log,
36+
BaseAdapter: NewBaseAdapter("github-copilot", projectID),
37+
sessionID: uuid.New().String(),
38+
hierarchy: hierarchyCache,
39+
log: log,
40+
projectIDInt: projID,
3541
}
3642
}
3743

@@ -114,7 +120,7 @@ func (a *CopilotAdapter) ParseLogFile(filePath string) ([]*types.AgentEvent, err
114120
// Extract workspace ID from path first
115121
// Path format: .../workspaceStorage/{workspace-id}/chatSessions/{session-id}.json
116122
workspaceID := extractWorkspaceIDFromPath(filePath)
117-
123+
118124
// Resolve hierarchy context if workspace ID found and hierarchy cache available
119125
var hierarchyCtx *hierarchy.WorkspaceContext
120126
if workspaceID != "" && a.hierarchy != nil {
@@ -123,11 +129,11 @@ func (a *CopilotAdapter) ParseLogFile(filePath string) ([]*types.AgentEvent, err
123129
a.log.Warnf("Failed to resolve workspace %s: %v - continuing without hierarchy", workspaceID, err)
124130
} else {
125131
hierarchyCtx = ctx
126-
a.log.Debugf("Resolved hierarchy for workspace %s: project=%d, machine=%d",
132+
a.log.Debugf("Resolved hierarchy for workspace %s: project=%d, machine=%d",
127133
workspaceID, ctx.ProjectID, ctx.MachineID)
128134
}
129135
}
130-
136+
131137
// Read the entire file
132138
data, err := os.ReadFile(filePath)
133139
if err != nil {
@@ -144,6 +150,9 @@ func (a *CopilotAdapter) ParseLogFile(filePath string) ([]*types.AgentEvent, err
144150
sessionID := extractSessionID(filePath)
145151
a.sessionID = sessionID
146152

153+
// Extract workspace ID from file path
154+
a.workspaceID = extractWorkspaceIDFromPath(filePath)
155+
147156
var events []*types.AgentEvent
148157

149158
// Process each request in the session
@@ -179,15 +188,15 @@ func extractSessionID(filePath string) string {
179188
func extractWorkspaceIDFromPath(filePath string) string {
180189
// Normalize path separators
181190
normalizedPath := filepath.ToSlash(filePath)
182-
191+
183192
// Look for workspaceStorage pattern
184193
parts := strings.Split(normalizedPath, "/")
185194
for i, part := range parts {
186195
if part == "workspaceStorage" && i+1 < len(parts) {
187196
return parts[i+1]
188197
}
189198
}
190-
199+
191200
return ""
192201
}
193202

@@ -260,12 +269,16 @@ func (a *CopilotAdapter) createLLMRequestEvent(
260269
Timestamp: timestamp,
261270
Type: types.EventTypeLLMRequest,
262271
AgentID: a.name,
272+
AgentVersion: "1.0.0",
263273
SessionID: a.sessionID,
274+
ProjectID: a.projectIDInt,
264275
LegacyProjectID: a.projectID, // Keep for backward compatibility
265276
Context: map[string]interface{}{
266277
"username": session.RequesterUsername,
267278
"location": session.InitialLocation,
268279
"variablesCount": len(request.VariableData.Variables),
280+
"workspaceId": a.workspaceID,
281+
"workspacePath": session.InitialLocation,
269282
},
270283
Data: map[string]interface{}{
271284
"requestId": request.RequestID,
@@ -279,7 +292,7 @@ func (a *CopilotAdapter) createLLMRequestEvent(
279292
}
280293

281294
// Add hierarchy context if available
282-
if hierarchyCtx != nil {
295+
if hierarchyCtx != nil && hierarchyCtx.ProjectID > 0 {
283296
event.ProjectID = hierarchyCtx.ProjectID
284297
event.MachineID = hierarchyCtx.MachineID
285298
event.WorkspaceID = hierarchyCtx.WorkspaceID
@@ -304,7 +317,9 @@ func (a *CopilotAdapter) createLLMResponseEvent(
304317
Timestamp: timestamp.Add(time.Second), // Slightly after request
305318
Type: types.EventTypeLLMResponse,
306319
AgentID: a.name,
320+
AgentVersion: "1.0.0",
307321
SessionID: a.sessionID,
322+
ProjectID: a.projectIDInt,
308323
LegacyProjectID: a.projectID,
309324
Data: map[string]interface{}{
310325
"requestId": request.RequestID,
@@ -318,7 +333,7 @@ func (a *CopilotAdapter) createLLMResponseEvent(
318333
}
319334

320335
// Add hierarchy context if available
321-
if hierarchyCtx != nil {
336+
if hierarchyCtx != nil && hierarchyCtx.ProjectID > 0 {
322337
event.ProjectID = hierarchyCtx.ProjectID
323338
event.MachineID = hierarchyCtx.MachineID
324339
event.WorkspaceID = hierarchyCtx.WorkspaceID
@@ -345,7 +360,9 @@ func (a *CopilotAdapter) createFileReferenceEvent(
345360
Timestamp: timestamp,
346361
Type: types.EventTypeFileRead,
347362
AgentID: a.name,
363+
AgentVersion: "1.0.0",
348364
SessionID: a.sessionID,
365+
ProjectID: a.projectIDInt,
349366
LegacyProjectID: a.projectID,
350367
Data: map[string]interface{}{
351368
"requestId": request.RequestID,
@@ -358,7 +375,7 @@ func (a *CopilotAdapter) createFileReferenceEvent(
358375
}
359376

360377
// Add hierarchy context if available
361-
if hierarchyCtx != nil {
378+
if hierarchyCtx != nil && hierarchyCtx.ProjectID > 0 {
362379
event.ProjectID = hierarchyCtx.ProjectID
363380
event.MachineID = hierarchyCtx.MachineID
364381
event.WorkspaceID = hierarchyCtx.WorkspaceID
@@ -399,7 +416,9 @@ func (a *CopilotAdapter) extractToolAndResponseEvents(
399416
Timestamp: timestamp.Add(timeOffset),
400417
Type: types.EventTypeFileRead,
401418
AgentID: a.name,
419+
AgentVersion: "1.0.0",
402420
SessionID: a.sessionID,
421+
ProjectID: a.projectIDInt,
403422
LegacyProjectID: a.projectID,
404423
Data: map[string]interface{}{
405424
"requestId": request.RequestID,
@@ -408,7 +427,7 @@ func (a *CopilotAdapter) extractToolAndResponseEvents(
408427
},
409428
}
410429
// Add hierarchy context if available
411-
if hierarchyCtx != nil {
430+
if hierarchyCtx != nil && hierarchyCtx.ProjectID > 0 {
412431
event.ProjectID = hierarchyCtx.ProjectID
413432
event.MachineID = hierarchyCtx.MachineID
414433
event.WorkspaceID = hierarchyCtx.WorkspaceID
@@ -423,15 +442,17 @@ func (a *CopilotAdapter) extractToolAndResponseEvents(
423442
Timestamp: timestamp.Add(timeOffset),
424443
Type: types.EventTypeFileModify,
425444
AgentID: a.name,
445+
AgentVersion: "1.0.0",
426446
SessionID: a.sessionID,
447+
ProjectID: a.projectIDInt,
427448
LegacyProjectID: a.projectID,
428449
Data: map[string]interface{}{
429450
"requestId": request.RequestID,
430451
"editCount": len(item.Edits),
431452
},
432453
}
433454
// Add hierarchy context if available
434-
if hierarchyCtx != nil {
455+
if hierarchyCtx != nil && hierarchyCtx.ProjectID > 0 {
435456
event.ProjectID = hierarchyCtx.ProjectID
436457
event.MachineID = hierarchyCtx.MachineID
437458
event.WorkspaceID = hierarchyCtx.WorkspaceID
@@ -478,13 +499,15 @@ func (a *CopilotAdapter) createToolInvocationEvent(
478499
Timestamp: timestamp,
479500
Type: types.EventTypeToolUse,
480501
AgentID: a.name,
502+
AgentVersion: "1.0.0",
481503
SessionID: a.sessionID,
504+
ProjectID: a.projectIDInt,
482505
LegacyProjectID: a.projectID,
483506
Data: data,
484507
}
485508

486509
// Add hierarchy context if available
487-
if hierarchyCtx != nil {
510+
if hierarchyCtx != nil && hierarchyCtx.ProjectID > 0 {
488511
event.ProjectID = hierarchyCtx.ProjectID
489512
event.MachineID = hierarchyCtx.MachineID
490513
event.WorkspaceID = hierarchyCtx.WorkspaceID

0 commit comments

Comments
 (0)