fix: wildcard metadata retrieves will now fetch source not in the project (#177)

* fix: wildcard retrieves will now fetch source not in the project

* chore: bump SDR to 4.1.1

* chore: update manifest create NUT for SDR changes

Author: shetzel

package.json

@@ -8,7 +8,7 @@
     "@oclif/config": "^1",
     "@salesforce/command": "^4.0.4",
     "@salesforce/core": "^2.26.1",
-    "@salesforce/source-deploy-retrieve": "^4.1.0",
+    "@salesforce/source-deploy-retrieve": "^4.1.1",
     "chalk": "^4.1.1",
     "cli-ux": "^5.6.3",
     "open": "^8.2.1",

src/componentSetBuilder.ts

@@ -7,7 +7,6 @@
 
 import * as path from 'path';
 import { ComponentSet, RegistryAccess } from '@salesforce/source-deploy-retrieve';
-import { ComponentLike } from '@salesforce/source-deploy-retrieve/lib/src/resolve/types';
 import { fs, SfdxError, Logger } from '@salesforce/core';
 
 export type ManifestOption = {
@@ -38,64 +37,69 @@ export class ComponentSetBuilder {
    */
   public static async build(options: ComponentSetOptions): Promise<ComponentSet> {
     const logger = Logger.childFromRoot('createComponentSet');
-    const csAggregator: ComponentLike[] = [];
+    let componentSet: ComponentSet;
 
     const { sourcepath, manifest, metadata, packagenames, apiversion, sourceapiversion } = options;
     try {
       if (sourcepath) {
         logger.debug(`Building ComponentSet from sourcepath: ${sourcepath.toString()}`);
+        const fsPaths: string[] = [];
         sourcepath.forEach((filepath) => {
-          if (fs.fileExistsSync(filepath)) {
-            csAggregator.push(...ComponentSet.fromSource(path.resolve(filepath)));
-          } else {
+          if (!fs.fileExistsSync(filepath)) {
             throw new SfdxError(`The sourcepath "${filepath}" is not a valid source file path.`);
           }
+          fsPaths.push(path.resolve(filepath));
         });
+        componentSet = ComponentSet.fromSource({ fsPaths });
       }
 
       // Return empty ComponentSet and use packageNames in the library via `.retrieve` options
       if (packagenames) {
         logger.debug(`Building ComponentSet for packagenames: ${packagenames.toString()}`);
-        csAggregator.push(...new ComponentSet([]));
+        componentSet ??= new ComponentSet();
       }
 
       // Resolve manifest with source in package directories.
       if (manifest) {
         logger.debug(`Building ComponentSet from manifest: ${manifest.manifestPath}`);
         const directoryPaths = options.manifest.directoryPaths;
         logger.debug(`Searching in packageDir: ${directoryPaths.join(', ')} for matching metadata`);
-        const compSet = await ComponentSet.fromManifest({
+        componentSet = await ComponentSet.fromManifest({
           manifestPath: manifest.manifestPath,
           resolveSourcePaths: options.manifest.directoryPaths,
           forceAddWildcards: true,
         });
-        csAggregator.push(...compSet);
       }
 
       // Resolve metadata entries with source in package directories.
       if (metadata) {
         logger.debug(`Building ComponentSet from metadata: ${metadata.metadataEntries.toString()}`);
         const registry = new RegistryAccess();
+        const compSetFilter = new ComponentSet();
+        componentSet ??= new ComponentSet();
 
         // Build a Set of metadata entries
-        const filter = new ComponentSet();
-        metadata.metadataEntries.forEach((entry) => {
-          const splitEntry = entry.split(':');
-          // try and get the type by name to ensure no typos or errors in type name
-          // matches toolbelt functionality
+        metadata.metadataEntries.forEach((rawEntry) => {
+          const splitEntry = rawEntry.split(':');
+          // The registry will throw if it doesn't know what this type is.
           registry.getTypeByName(splitEntry[0]);
-          filter.add({
+          const entry = {
             type: splitEntry[0],
             fullName: splitEntry.length === 1 ? '*' : splitEntry[1],
-          });
+          };
+          // Add to the filtered ComponentSet for resolved source paths,
+          // and the unfiltered ComponentSet to build the correct manifest.
+          compSetFilter.add(entry);
+          componentSet.add(entry);
         });
 
         const directoryPaths = options.metadata.directoryPaths;
         logger.debug(`Searching for matching metadata in directories: ${directoryPaths.join(', ')}`);
-        const fromSource = ComponentSet.fromSource({ fsPaths: directoryPaths, include: filter });
-        // If no matching metadata is found, default to the original component set
-        const finalized = fromSource.size > 0 ? fromSource : filter;
-        csAggregator.push(...finalized);
+        const resolvedComponents = ComponentSet.fromSource({ fsPaths: directoryPaths, include: compSetFilter });
+        componentSet.forceIgnoredPaths = resolvedComponents.forceIgnoredPaths;
+        for (const comp of resolvedComponents) {
+          componentSet.add(comp);
+        }
       }
     } catch (e) {
       if ((e as Error).message.includes('Missing metadata type definition in registry for id')) {
@@ -108,8 +112,6 @@ export class ComponentSetBuilder {
       }
     }
 
-    const componentSet = new ComponentSet(csAggregator);
-
     // This is only for debug output of matched files based on the command flags.
     // It will log up to 20 file matches.
     if (logger.debugEnabled && componentSet.size) {

test/commands/source/componentSetBuilder.test.ts

@@ -58,8 +58,8 @@ describe('ComponentSetBuilder', () => {
         metadata: undefined,
       });
 
-      const expectedPath = path.resolve(sourcepath[0]);
-      expect(fromSourceStub.calledOnceWith(expectedPath)).to.equal(true);
+      const expectedArg = { fsPaths: [path.resolve(sourcepath[0])] };
+      expect(fromSourceStub.calledOnceWith(expectedArg)).to.equal(true);
       expect(compSet.size).to.equal(1);
       expect(compSet.has(apexClassComponent)).to.equal(true);
     });
@@ -78,9 +78,8 @@ describe('ComponentSetBuilder', () => {
       });
       const expectedPath1 = path.resolve(sourcepath[0]);
       const expectedPath2 = path.resolve(sourcepath[1]);
-      expect(fromSourceStub.calledTwice).to.equal(true);
-      expect(fromSourceStub.firstCall.args[0]).to.equal(expectedPath1);
-      expect(fromSourceStub.secondCall.args[0]).to.equal(expectedPath2);
+      const expectedArg = { fsPaths: [expectedPath1, expectedPath2] };
+      expect(fromSourceStub.calledOnceWith(expectedArg)).to.equal(true);
       expect(compSet.size).to.equal(2);
       expect(compSet.has(apexClassComponent)).to.equal(true);
       expect(compSet.has(customObjectComponent)).to.equal(true);
@@ -98,8 +97,8 @@ describe('ComponentSetBuilder', () => {
       };
 
       const compSet = await ComponentSetBuilder.build(options);
-      const expectedPath = path.resolve(sourcepath[0]);
-      expect(fromSourceStub.calledOnceWith(expectedPath)).to.equal(true);
+      const expectedArg = { fsPaths: [path.resolve(sourcepath[0])] };
+      expect(fromSourceStub.calledOnceWith(expectedArg)).to.equal(true);
       expect(compSet.size).to.equal(0);
       expect(compSet.apiVersion).to.equal(options.apiversion);
     });
@@ -116,8 +115,8 @@ describe('ComponentSetBuilder', () => {
       };
 
       const compSet = await ComponentSetBuilder.build(options);
-      const expectedPath = path.resolve(sourcepath[0]);
-      expect(fromSourceStub.calledOnceWith(expectedPath)).to.equal(true);
+      const expectedArg = { fsPaths: [path.resolve(sourcepath[0])] };
+      expect(fromSourceStub.calledOnceWith(expectedArg)).to.equal(true);
       expect(compSet.size).to.equal(0);
       expect(compSet.sourceApiVersion).to.equal(options.sourceapiversion);
     });
@@ -173,8 +172,9 @@ describe('ComponentSetBuilder', () => {
       filter.add({ type: 'ApexClass', fullName: '*' });
       expect(fromSourceArgs).to.have.property('include');
       expect(fromSourceArgs.include.getSourceComponents()).to.deep.equal(filter.getSourceComponents());
-      expect(compSet.size).to.equal(1);
+      expect(compSet.size).to.equal(2);
       expect(compSet.has(apexClassComponent)).to.equal(true);
+      expect(compSet.has({ type: 'ApexClass', fullName: '*' })).to.equal(true);
     });
 
     it('should throw an error when it cant resolve a metadata type (Metadata)', async () => {
@@ -242,9 +242,10 @@ describe('ComponentSetBuilder', () => {
       filter.add({ type: 'CustomObject', fullName: '*' });
       expect(fromSourceArgs).to.have.property('include');
       expect(fromSourceArgs.include.getSourceComponents()).to.deep.equal(filter.getSourceComponents());
-      expect(compSet.size).to.equal(2);
+      expect(compSet.size).to.equal(3);
       expect(compSet.has(apexClassComponent)).to.equal(true);
       expect(compSet.has(customObjectComponent)).to.equal(true);
+      expect(compSet.has({ type: 'CustomObject', fullName: '*' })).to.equal(true);
     });
 
     it('should create ComponentSet from metadata and multiple package directories', async () => {
@@ -270,9 +271,10 @@ describe('ComponentSetBuilder', () => {
       filter.add({ type: 'ApexClass', fullName: '*' });
       expect(fromSourceArgs).to.have.property('include');
       expect(fromSourceArgs.include.getSourceComponents()).to.deep.equal(filter.getSourceComponents());
-      expect(compSet.size).to.equal(2);
+      expect(compSet.size).to.equal(3);
       expect(compSet.has(apexClassComponent)).to.equal(true);
       expect(compSet.has(apexClassComponent2)).to.equal(true);
+      expect(compSet.has({ type: 'ApexClass', fullName: '*' })).to.equal(true);
     });
 
     it('should create ComponentSet from manifest', async () => {

test/nuts/create.nut.ts

@@ -16,6 +16,7 @@ const apexManifest =
   '<?xml version="1.0" encoding="UTF-8"?>\n' +
   '<Package xmlns="http://soap.sforce.com/2006/04/metadata">\n' +
   '    <types>\n' +
+  '        <members>*</members>\n' +
   '        <members>GeocodingService</members>\n' +
   '        <members>GeocodingServiceTest</members>\n' +
   '        <members>PagedResult</members>\n' +

yarn.lock

@@ -843,10 +843,10 @@
     unzipper "0.10.11"
     xmldom-sfdx-encoding "^0.1.29"
 
-"@salesforce/source-deploy-retrieve@^4.1.0":
-  version "4.1.0"
-  resolved "https://registry.npmjs.org/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-4.1.0.tgz#b28b8e1c9c810884315c564751e8854c4e1e16d3"
-  integrity sha512-dLdCNs8KZl2AlI5nxzqjPFTt/RiOQqlbIFsmsbnY/i0Vk15/Fqyxu1cBU0stV3+NxHYV/ijzz2jRVq87G7+jRg==
+"@salesforce/source-deploy-retrieve@^4.1.1":
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-4.1.1.tgz#8c5b23cec488dcd4aea5e15df49e014bf00a7929"
+  integrity sha512-tfuPtmdtWU/1HwuB+ZM6gvCXUBZh+XOiBLPz98KU3khKi81CRnJyXQCL+tiBZpIwzGaQV1emOIZD2WsjTEY9oA==
   dependencies:
     "@salesforce/core" "2.25.1"
     "@salesforce/kit" "^1.5.0"