feat(backend): enhance pagination handling and logging for registry sync

Author: Lasim

services/backend/src/services/registrySyncService.ts

@@ -259,6 +259,14 @@ export class RegistrySyncService {
 
       const { servers, metadata } = batchResult.data;
 
+      // Log the actual metadata structure to debug pagination issues
+      this.logger.debug({
+        pageNumber,
+        metadataKeys: Object.keys(metadata || {}),
+        metadataStructure: JSON.stringify(metadata, null, 2),
+        operation: 'registry_api_metadata'
+      }, 'Registry API metadata structure');
+      
       // Extract server data and filter for isLatest === true (CRITICAL: keep this filter!)
       const serverData = servers
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -279,7 +287,7 @@ export class RegistrySyncService {
         pageNumber,
         rawBatchSize: servers.length,
         afterIsLatestFilter: serverData.length,
-        cursor,
+        currentCursor: cursor,
       }, 'Fetched and filtered page from official registry');
 
       // Filter this page against database (only check these 50 servers)
@@ -296,9 +304,36 @@ export class RegistrySyncService {
       // Add new servers from this page to accumulated list
       accumulatedNewServers.push(...newServersInPage);
 
-      // Check for next page
-      cursor = metadata.next_cursor;
-      hasMore = !!cursor;
+      // Check for next page - try multiple possible field names
+      // Official MCP Registry might use different naming conventions
+      cursor = metadata.next_cursor || metadata.nextCursor || metadata.cursor || metadata.next;
+      
+      // Determine if there are more pages
+      // Stop if: no cursor OR we fetched fewer servers than requested (end of data)
+      const shouldContinue = !!cursor && servers.length > 0;
+      hasMore = shouldContinue;
+      
+      this.logger.debug({
+        pageNumber,
+        nextCursor: cursor,
+        serversInBatch: servers.length,
+        hasMore,
+        metadataCount: metadata.count,
+        metadataTotal: metadata.total,
+        accumulated: accumulatedNewServers.length,
+        targetMax: maxServers,
+        operation: 'pagination_check'
+      }, 'Pagination status after page fetch');
+      
+      // If we have no cursor but metadata suggests more servers exist, log a warning
+      if (!cursor && metadata.total && metadata.count < metadata.total) {
+        this.logger.warn({
+          pageNumber,
+          metadataCount: metadata.count,
+          metadataTotal: metadata.total,
+          operation: 'pagination_cursor_missing'
+        }, 'API indicates more servers exist but no cursor provided - possible API issue');
+      }
 
       // Rate limiting for registry API
       if (hasMore) {

services/backend/src/services/transforms/officialRegistryTransforms.ts

@@ -46,6 +46,8 @@ export interface OfficialEnvironmentVariable {
  */
 export interface OfficialTransport {
   type: 'stdio' | 'streamable-http' | 'sse';
+  command?: string; // Optional in official registry
+  args?: string[]; // Optional in official registry
 }
 
 /**
@@ -176,6 +178,62 @@ export function createSlug(officialName: string): string {
 // TRANSPORT TYPE DERIVATION
 // =============================================================================
 
+/**
+ * Infer command and args for STDIO packages based on registry type
+ * 
+ * The official MCP registry doesn't always provide command/args in the transport object.
+ * We need to infer them based on the package's registryType and identifier.
+ * 
+ * @param pkg - Official package data
+ * @returns Inferred transport with command and args
+ */
+export function inferStdioTransport(pkg: OfficialPackage): {
+  command: string;
+  args: string[];
+} {
+  // If transport already has command and args, use them
+  if (pkg.transport.command && pkg.transport.args) {
+    return {
+      command: pkg.transport.command,
+      args: pkg.transport.args
+    };
+  }
+  
+  // Otherwise, infer based on registryType
+  const identifier = pkg.identifier;
+  
+  switch (pkg.registryType.toLowerCase()) {
+    case 'npm':
+      // NPM packages: npx -y <package-name>
+      return {
+        command: 'npx',
+        args: ['-y', identifier]
+      };
+    
+    case 'pypi':
+      // PyPI packages: uvx <package-name>
+      // Alternative: python -m <package-name>
+      return {
+        command: 'uvx',
+        args: [identifier]
+      };
+    
+    case 'docker':
+      // Docker images: docker run <image-name>
+      return {
+        command: 'docker',
+        args: ['run', identifier]
+      };
+    
+    default:
+      // Fallback to npx for unknown types
+      return {
+        command: 'npx',
+        args: [identifier]
+      };
+  }
+}
+
 /**
  * Derive DeployStack transport_type from official packages/remotes
  * 
@@ -365,6 +423,8 @@ export function mapHeadersToThreeTier(
 /**
  * Map official runtime/package arguments to DeployStack's 3-tier args system
  * 
+ * IMPORTANT: For STDIO packages, we need to infer command/args if not provided
+ * 
  * @param packages - Official packages array
  * @returns DeployStack ConfigurationSchema args configuration
  */
@@ -382,6 +442,29 @@ export function mapArgumentsToThreeTier(
   // Extract runtime and package arguments
   const pkg = packages[0];
 
+  // For STDIO packages, infer transport command/args if not provided
+  if (pkg.transport.type === 'stdio') {
+    const inferredTransport = inferStdioTransport(pkg);
+    
+    // Add inferred args as template args (locked)
+    for (const arg of inferredTransport.args) {
+      templateArgs.push({
+        value: arg,
+        locked: true,
+        description: `Static argument: ${arg}`
+      });
+    }
+    
+    // Store command in the package for later use
+    // This will be used when constructing the packages array
+    if (!pkg.transport.command) {
+      pkg.transport.command = inferredTransport.command;
+    }
+    if (!pkg.transport.args) {
+      pkg.transport.args = inferredTransport.args;
+    }
+  }
+  
   // Package arguments go to template (locked)
   if (pkg.packageArguments && pkg.packageArguments.length > 0) {
     for (const arg of pkg.packageArguments) {
@@ -435,12 +518,17 @@ export async function transformOfficialToDeployStack(
     fetchGitHubMetadata?: boolean;
   }
 ): Promise<Partial<CreateGlobalServerRequest>> {
+  // IMPORTANT: Create a deep copy of packages to avoid mutating the original
+  // We'll be adding inferred command/args to the packages during transformation
+  const packagesCopy = officialServer.packages ? JSON.parse(JSON.stringify(officialServer.packages)) : undefined;
+  
   // Extract 3-tier configurations from packages (env vars + args)
   const envConfig = mapEnvironmentVariablesToThreeTier(
-    officialServer.packages?.[0]?.environmentVariables || []
+    packagesCopy?.[0]?.environmentVariables || []
   );
 
-  const argsConfig = mapArgumentsToThreeTier(officialServer.packages);
+  // This will infer and add command/args to packagesCopy if missing
+  const argsConfig = mapArgumentsToThreeTier(packagesCopy);
 
   // Extract 3-tier configurations from remotes (headers)
   // Merge headers from all remotes (in case there are multiple endpoints)
@@ -500,7 +588,8 @@ export async function transformOfficialToDeployStack(
     website_url: officialServer.websiteUrl,
 
     // Official format storage (will be JSON stringified by create-global.ts)
-    packages: officialServer.packages,
+    // Use packagesCopy which now has inferred command/args
+    packages: packagesCopy,
     remotes: officialServer.remotes,
 
     // Derived DeployStack fields