Merge pull request continuedev#5059 from continuedev/nate/local-blocks

Nate/local-blocks

Author: sestinj

.github/workflows/pr_checks.yaml

@@ -1,5 +1,6 @@
 name: PR checks
 
+# PR Checks run on all PRs to the main branch and must pass before merging.
 on:
   pull_request:
     branches:
@@ -415,6 +416,30 @@ jobs:
           AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
           AZURE_FOUNDRY_API_KEY: ${{ secrets.AZURE_FOUNDRY_API_KEY }}
 
+  package-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+
+      - uses: actions/cache@v4
+        with:
+          path: core/node_modules
+          key: ${{ runner.os }}-config-yaml-node-modules-${{ hashFiles('packages/config-yaml/package-lock.json') }}
+
+      - name: Install dependencies
+        run: |
+          cd packages/config-yaml
+          npm ci
+
+      - name: Run config-yaml tests
+        run: |
+          cd packages/config-yaml
+          npm test
+
   vscode-get-test-file-matrix:
     runs-on: ubuntu-latest
     needs: [install-root, install-vscode, install-config-yaml]
@@ -551,7 +576,7 @@ jobs:
           path: extensions/vscode/e2e/storage
 
   vscode-e2e-tests:
-    name: ${{ matrix.test_file }} (${{ matrix.command }})
+    name: ${{ matrix.test_file || 'unknown' }} (${{ matrix.command }})
     needs:
       [
         vscode-download-e2e-dependencies,
@@ -652,6 +677,27 @@ jobs:
           name: e2e-failure-screenshots
           path: extensions/vscode/e2e/storage/screenshots
 
+      - name: Sanitize test file name
+        id: sanitize_filename
+        if: always()
+        run: |
+          FILENAME="${{ matrix.test_file }}"
+          SANITIZED_FILENAME="${FILENAME//\//-}" # Replace / with - using bash parameter expansion
+          echo "sanitized_test_file=${SANITIZED_FILENAME}" >> $GITHUB_OUTPUT
+
+      - name: Find e2e log file
+        if: always()
+        run: |
+          echo "Searching for e2e.log file..."
+          find $GITHUB_WORKSPACE -name "e2e.log" -type f
+
+      - name: Upload e2e logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-logs-${{ steps.sanitize_filename.outputs.sanitized_test_file || 'unknown' }}-${{ matrix.command == 'e2e:ci:run-yaml' && 'yaml' || 'json' }}
+          path: extensions/vscode/e2e.log
+
   gui-tests:
     needs: [install-gui, install-core, install-config-yaml]
     runs-on: ubuntu-latest
@@ -820,6 +866,7 @@ jobs:
       - install-vscode
       - vscode-checks
       - core-tests
+      - package-tests
       - vscode-get-test-file-matrix
       - vscode-package-extension
       - vscode-download-e2e-dependencies

.gitignore

@@ -83,16 +83,6 @@ ipython_config.py
 # pyenv
 .python-version
 
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
 # Celery stuff
 celerybeat-schedule
 celerybeat.pid

.idea/compiler.xml

@@ -15,7 +15,12 @@ import {
 } from "../index.js";
 import { GlobalContext } from "../util/GlobalContext.js";
 
-import { getAllAssistantFiles } from "./loadLocalAssistants.js";
+import { logger } from "../util/logger.js";
+import {
+  ASSISTANTS,
+  getAllDotContinueYamlFiles,
+  LoadAssistantFilesOptions,
+} from "./loadLocalAssistants.js";
 import LocalProfileLoader from "./profile/LocalProfileLoader.js";
 import PlatformProfileLoader from "./profile/PlatformProfileLoader.js";
 import {
@@ -185,7 +190,11 @@ export class ConfigHandler {
   private async getNonPersonalHubOrg(
     org: OrganizationDescription,
   ): Promise<OrgWithProfiles> {
-    const profiles = await this.getHubProfiles(org.id);
+    const localProfiles = await this.getLocalProfiles({
+      includeGlobal: false,
+      includeWorkspace: true,
+    });
+    const profiles = [...(await this.getHubProfiles(org.id)), ...localProfiles];
     return this.rectifyProfilesForOrg(org, profiles);
   }
 
@@ -196,15 +205,21 @@ export class ConfigHandler {
     slug: undefined,
   };
   private async getPersonalHubOrg() {
-    const allLocalProfiles = await this.getAllLocalProfiles();
+    const localProfiles = await this.getLocalProfiles({
+      includeGlobal: true,
+      includeWorkspace: true,
+    });
     const hubProfiles = await this.getHubProfiles(null);
-    const profiles = [...hubProfiles, ...allLocalProfiles];
+    const profiles = [...hubProfiles, ...localProfiles];
     return this.rectifyProfilesForOrg(this.PERSONAL_ORG_DESC, profiles);
   }
 
   private async getLocalOrg() {
-    const allLocalProfiles = await this.getAllLocalProfiles();
-    return this.rectifyProfilesForOrg(this.PERSONAL_ORG_DESC, allLocalProfiles);
+    const localProfiles = await this.getLocalProfiles({
+      includeGlobal: true,
+      includeWorkspace: true,
+    });
+    return this.rectifyProfilesForOrg(this.PERSONAL_ORG_DESC, localProfiles);
   }
 
   private async rectifyProfilesForOrg(
@@ -251,24 +266,41 @@ export class ConfigHandler {
     };
   }
 
-  async getAllLocalProfiles() {
+  async getLocalProfiles(options: LoadAssistantFilesOptions) {
     /**
      * Users can define as many local assistants as they want in a `.continue/assistants` folder
      */
-    const assistantFiles = await getAllAssistantFiles(this.ide);
-    const profiles = assistantFiles.map((assistant) => {
-      return new LocalProfileLoader(
+    const localProfiles: ProfileLifecycleManager[] = [];
+
+    if (options.includeGlobal) {
+      localProfiles.push(this.globalLocalProfileManager);
+    }
+
+    if (options.includeWorkspace) {
+      const assistantFiles = await getAllDotContinueYamlFiles(
         this.ide,
-        this.ideSettingsPromise,
-        this.controlPlaneClient,
-        this.llmLogger,
-        assistant,
+        {
+          ...options,
+          includeGlobal: false, // Because the global comes from above
+        },
+        ASSISTANTS,
       );
-    });
-    const localAssistantProfiles = profiles.map(
-      (profile) => new ProfileLifecycleManager(profile, this.ide),
-    );
-    return [this.globalLocalProfileManager, ...localAssistantProfiles];
+      const profiles = assistantFiles.map((assistant) => {
+        return new LocalProfileLoader(
+          this.ide,
+          this.ideSettingsPromise,
+          this.controlPlaneClient,
+          this.llmLogger,
+          assistant,
+        );
+      });
+      const localAssistantProfiles = profiles.map(
+        (profile) => new ProfileLifecycleManager(profile, this.ide),
+      );
+      localProfiles.push(...localAssistantProfiles);
+    }
+
+    return localProfiles;
   }
 
   //////////////////
@@ -421,9 +453,14 @@ export class ConfigHandler {
         configLoadInterrupted: true,
       };
     }
-    return await this.currentProfile.loadConfig(
+    const config = await this.currentProfile.loadConfig(
       this.additionalContextProviders,
     );
+
+    if (config.errors?.length) {
+      logger.warn("Errors loading config: ", config.errors);
+    }
+    return config;
   }
 
   async openConfigProfile(profileId?: string) {

core/config/ConfigHandler.ts

@@ -95,7 +95,10 @@ export class ProfileLifecycleManager {
       try {
         result = await this.profileLoader.doLoadConfig();
       } catch (e) {
-        const message = e instanceof Error ? e.message : "Error loading config";
+        const message =
+          e instanceof Error
+            ? `${e.message}\n${e.stack ? e.stack : ""}`
+            : "Error loading config";
         result = {
           errors: [
             {

core/config/ProfileLifecycleManager.ts

@@ -5,11 +5,12 @@ import {
   DEFAULT_IGNORE_FILETYPES,
 } from "../indexing/ignore";
 import { walkDir } from "../indexing/walkDir";
-import { getGlobalAssistantsPath } from "../util/paths";
+import { getGlobalFolderWithName } from "../util/paths";
 import { localPathToUri } from "../util/pathToUri";
 import { joinPathsToUri } from "../util/uri";
 
-export const ASSISTANTS_FOLDER = ".continue/assistants";
+export const ASSISTANTS = "assistants";
+export const ASSISTANTS_FOLDER = `.continue/${ASSISTANTS}`;
 
 export function isLocalAssistantFile(uri: string): boolean {
   if (!uri.endsWith(".yaml") && !uri.endsWith(".yml")) {
@@ -20,7 +21,7 @@ export function isLocalAssistantFile(uri: string): boolean {
   return normalizedUri.includes(`/${ASSISTANTS_FOLDER}/`);
 }
 
-export async function getAssistantFilesFromDir(
+export async function listYamlFilesInDir(
   ide: IDE,
   dir: string,
 ): Promise<{ path: string; content: string }[]> {
@@ -53,21 +54,56 @@ export async function getAssistantFilesFromDir(
   }
 }
 
-export async function getAllAssistantFiles(
+export interface LoadAssistantFilesOptions {
+  includeGlobal: boolean;
+  includeWorkspace: boolean;
+}
+
+export function getDotContinueSubDirs(
   ide: IDE,
+  options: LoadAssistantFilesOptions,
+  workspaceDirs: string[],
+  subDirName: string,
+): string[] {
+  let fullDirs: string[] = [];
+
+  // Workspace .continue/<subDirName>
+  if (options.includeWorkspace) {
+    fullDirs = workspaceDirs.map((dir) =>
+      joinPathsToUri(dir, ".continue", subDirName),
+    );
+  }
+
+  // ~/.continue/<subDirName>
+  if (options.includeGlobal) {
+    fullDirs.push(localPathToUri(getGlobalFolderWithName(subDirName)));
+  }
+
+  return fullDirs;
+}
+
+/**
+ * This method searches in both ~/.continue and workspace .continue
+ * for all YAML files in the specified subdirctory, for example .continue/assistants or .continue/prompts
+ */
+export async function getAllDotContinueYamlFiles(
+  ide: IDE,
+  options: LoadAssistantFilesOptions,
+  subDirName: string,
 ): Promise<{ path: string; content: string }[]> {
   const workspaceDirs = await ide.getWorkspaceDirs();
-  let assistantFiles: { path: string; content: string }[] = [];
-
-  let dirsToCheck = [ASSISTANTS_FOLDER];
-  const fullDirs = workspaceDirs
-    .map((dir) => dirsToCheck.map((d) => joinPathsToUri(dir, d)))
-    .flat();
 
-  fullDirs.push(localPathToUri(getGlobalAssistantsPath()));
+  // Get all directories to check for assistant files
+  const fullDirs = getDotContinueSubDirs(
+    ide,
+    options,
+    workspaceDirs,
+    subDirName,
+  );
 
-  assistantFiles = (
-    await Promise.all(fullDirs.map((dir) => getAssistantFilesFromDir(ide, dir)))
+  // Get all assistant files from the directories
+  const assistantFiles = (
+    await Promise.all(fullDirs.map((dir) => listYamlFilesInDir(ide, dir)))
   ).flat();
 
   return await Promise.all(

core/config/loadLocalAssistants.ts

@@ -60,12 +60,14 @@ export default class LocalProfileLoader implements IProfileLoader {
       ideSettingsPromise: this.ideSettingsPromise,
       controlPlaneClient: this.controlPlaneClient,
       llmLogger: this.llmLogger,
-      overrideConfigJson: undefined,
-      overrideConfigYaml: undefined,
-      platformConfigMetadata: undefined,
       profileId: this.description.id,
       overrideConfigYamlByPath: this.overrideAssistantFile?.path,
       orgScopeId: null,
+      packageIdentifier: {
+        uriType: "file",
+        filePath:
+          this.overrideAssistantFile?.path ?? getPrimaryConfigFilePath(),
+      },
     });
 
     this.description.errors = result.errors;

core/config/profile/LocalProfileLoader.ts

@@ -142,15 +142,17 @@ export default class PlatformProfileLoader implements IProfileLoader {
       ideSettingsPromise: this.ideSettingsPromise,
       controlPlaneClient: this.controlPlaneClient,
       llmLogger: this.llmLogger,
-      overrideConfigJson: undefined,
       overrideConfigYaml: this.configResult.config,
-      platformConfigMetadata: {
-        ownerSlug: this.ownerSlug,
-        packageSlug: this.packageSlug,
-      },
       profileId: this.description.id,
-      overrideConfigYamlByPath: undefined,
       orgScopeId: this.orgScopeId,
+      packageIdentifier: {
+        uriType: "slug",
+        fullSlug: {
+          ownerSlug: this.ownerSlug,
+          packageSlug: this.packageSlug,
+          versionSlug: this.versionSlug,
+        },
+      },
     });
 
     return {