55
66import * as vscode from 'vscode'
77import * as path from 'path'
8- import { SystemUtilities } from '../../shared/systemUtilities'
9- import { getGlobDirExcludedPatterns } from '../../shared/fs/watchedFiles'
10- import { getWorkspaceRelativePath } from '../../shared/utilities/workspaceUtils'
11- import { Uri } from 'vscode'
12- import { GitIgnoreFilter } from './gitignore'
8+ import { collectFiles } from '../../shared/utilities/workspaceUtils'
139
1410import AdmZip from 'adm-zip'
1511import { ContentLengthError , PrepareRepoFailedError } from '../errors'
@@ -20,131 +16,6 @@ import { CurrentWsFolders } from '../types'
2016import { ToolkitError } from '../../shared/errors'
2117import { AmazonqCreateUpload , Metric } from '../../shared/telemetry/telemetry'
2218import { TelemetryHelper } from './telemetryHelper'
23- import { sanitizeFilename } from '../../shared/utilities/textUtilities'
24- import { maxRepoSizeBytes } from '../constants'
25-
26- export function getExcludePattern ( additionalPatterns : string [ ] = [ ] ) {
27- const globAlwaysExcludedDirs = getGlobDirExcludedPatterns ( ) . map ( pattern => `**/${ pattern } )
28- const extraPatterns = [
29- '**/package-lock.json' ,
30- '**/yarn.lock' ,
31- '**/*.zip' ,
32- '**/*.bin' ,
33- '**/*.png' ,
34- '**/*.jpg' ,
35- '**/*.svg' ,
36- '**/*.pyc' ,
37- '**/license.txt' ,
38- '**/License.txt' ,
39- '**/LICENSE.txt' ,
40- '**/license.md' ,
41- '**/License.md' ,
42- '**/LICENSE.md' ,
43- ]
44- const allPatterns = [ ...globAlwaysExcludedDirs , ...extraPatterns , ...additionalPatterns ]
45- return `{${ allPatterns . join ( ',' ) }
46- }
47-
48- /**
49- * @param rootPath root folder to look for .gitignore files
50- * @returns list of glob patterns extracted from .gitignore
51- * These patterns are compatible with vscode exclude patterns
52- */
53- async function filterOutGitignoredFiles ( rootPath : string , files : Uri [ ] ) : Promise < Uri [ ] > {
54- const gitIgnoreFiles = await vscode . workspace . findFiles (
55- new vscode . RelativePattern ( rootPath , '**/.gitignore' ) ,
56- getExcludePattern ( )
57- )
58- const gitIgnoreFilter = await GitIgnoreFilter . build ( gitIgnoreFiles )
59- return gitIgnoreFilter . filterFiles ( files )
60- }
61-
62- /**
63- * collects all files that are marked as source
64- * @param sourcePaths the paths where collection starts
65- * @param workspaceFolders the current workspace folders opened
66- * @param respectGitIgnore whether to respect gitignore file
67- * @returns all matched files
68- */
69- export async function collectFiles (
70- sourcePaths : string [ ] ,
71- workspaceFolders : CurrentWsFolders ,
72- respectGitIgnore : boolean = true
73- ) : Promise <
74- {
75- workspaceFolder : vscode . WorkspaceFolder
76- relativeFilePath : string
77- fileUri : vscode . Uri
78- fileContent : string
79- zipFilePath : string
80- } [ ]
81- > {
82- const storage : Awaited < ReturnType < typeof collectFiles > > = [ ]
83-
84- const workspaceFoldersMapping = getWorkspaceFoldersByPrefixes ( workspaceFolders )
85- const workspaceToPrefix = new Map < vscode . WorkspaceFolder , string > (
86- workspaceFoldersMapping === undefined
87- ? [ [ workspaceFolders [ 0 ] , '' ] ]
88- : Object . entries ( workspaceFoldersMapping ) . map ( value => [ value [ 1 ] , value [ 0 ] ] )
89- )
90- const prefixWithFolderPrefix = ( folder : vscode . WorkspaceFolder , path : string ) => {
91- const prefix = workspaceToPrefix . get ( folder )
92- if ( prefix === undefined ) {
93- throw new ToolkitError ( `Failed to find prefix for workspace folder ${ folder . name } )
94- }
95- return prefix === '' ? path : `${ prefix } ${ path }
96- }
97-
98- let totalSizeBytes = 0
99- for ( const rootPath of sourcePaths ) {
100- const allFiles = await vscode . workspace . findFiles (
101- new vscode . RelativePattern ( rootPath , '**' ) ,
102- getExcludePattern ( )
103- )
104- const files = respectGitIgnore ? await filterOutGitignoredFiles ( rootPath , allFiles ) : allFiles
105-
106- for ( const file of files ) {
107- const relativePath = getWorkspaceRelativePath ( file . fsPath , { workspaceFolders } )
108- if ( ! relativePath ) {
109- continue
110- }
111-
112- const fileStat = await vscode . workspace . fs . stat ( file )
113- if ( totalSizeBytes + fileStat . size > maxRepoSizeBytes ) {
114- throw new ContentLengthError ( )
115- }
116-
117- const fileContent = await readFile ( file )
118- if ( fileContent === undefined ) {
119- continue
120- }
121-
122- // Now that we've read the file, increase our usage
123- totalSizeBytes += fileStat . size
124- storage . push ( {
125- workspaceFolder : relativePath . workspaceFolder ,
126- relativeFilePath : relativePath . relativePath ,
127- fileUri : file ,
128- fileContent : fileContent ,
129- zipFilePath : prefixWithFolderPrefix ( relativePath . workspaceFolder , relativePath . relativePath ) ,
130- } )
131- }
132- }
133- return storage
134- }
135-
136- const readFile = async ( file : vscode . Uri ) => {
137- try {
138- const fileContent = await SystemUtilities . readFile ( file , new TextDecoder ( 'utf8' , { fatal : false } ) )
139- return fileContent
140- } catch ( error ) {
141- getLogger ( ) . debug (
142- `featureDev: Failed to read file ${ file . fsPath }
143- )
144- }
145-
146- return undefined
147- }
14819
14920const getSha256 = ( file : Buffer ) => createHash ( 'sha256' ) . update ( file ) . digest ( 'base64' )
15021
@@ -191,123 +62,6 @@ export async function prepareRepoData(
19162 }
19263}
19364
194- const workspaceFolderPrefixGuards = {
195- /**
196- * the maximum number of subfolders the method below takes into account when calculating a prefix
197- */
198- maximumFolderDepthConsidered : 500 ,
199- /**
200- * the maximum suffix that can be added to a folder prefix in case of full subfolder path matches
201- */
202- maximumFoldersWithMatchingSubfolders : 10_000 ,
203- }
204-
205- /**
206- * tries to determine the possible prefixes we will use for a given workspace folder in the zip file
207- * We want to keep the folder names in the prefix, since they might convey useful information, for example
208- * If both folders are just called cdk (no name specified for the ws folder), adding a prefix of cdk1 and cdk2 is much less context, than having app_cdk and canaries_cdk
209- *
210- * Input:
211- * - packages/app/cdk
212- * - packages/canaries/cdk
213- * Output:
214- * - {'app_cdk': packages/app/cdk, 'canaries_cdk': packages/canaries/cdk}
215- *
216- * @returns an object where workspace folders have a prefix, or undefined for single root workspace, as there is no mapping needed there
217- */
218- export function getWorkspaceFoldersByPrefixes (
219- folders : CurrentWsFolders
220- ) : { [ prefix : string ] : vscode . WorkspaceFolder } | undefined {
221- if ( folders . length <= 1 ) {
222- return undefined
223- }
224- let remainingWorkspaceFoldersToMap = folders . map ( f => ( {
225- folder : f ,
226- preferredPrefixQueue : f . uri . fsPath
227- . split ( path . sep )
228- . reverse ( )
229- . slice ( 0 , workspaceFolderPrefixGuards . maximumFolderDepthConsidered )
230- . reduce (
231- ( candidates , subDir ) => {
232- candidates . push ( sanitizeFilename ( path . join ( subDir , candidates [ candidates . length - 1 ] ) ) )
233- return candidates
234- } ,
235- [ f . name ]
236- )
237- . reverse ( ) ,
238- } ) )
239- const results : ReturnType < typeof getWorkspaceFoldersByPrefixes > = { }
240-
241- for (
242- let addParentFolderCount = 0 ;
243- remainingWorkspaceFoldersToMap . length > 0 &&
244- addParentFolderCount < workspaceFolderPrefixGuards . maximumFolderDepthConsidered ;
245- addParentFolderCount ++
246- ) {
247- const workspacesByPrefixes = remainingWorkspaceFoldersToMap . reduce ( ( acc , wsFolder ) => {
248- const prefix = wsFolder . preferredPrefixQueue . pop ( )
249- // this should never happen, as last candidates should be handled below, and the array starts non empty
250- if ( prefix === undefined ) {
251- throw new ToolkitError (
252- `Encountered a folder with invalid prefix candidates (workspace folder ${ wsFolder . folder . name }
253- )
254- }
255- acc [ prefix ] = acc [ prefix ] ?? [ ]
256- acc [ prefix ] . push ( wsFolder )
257- return acc
258- } , { } as { [ key : string ] : ( typeof remainingWorkspaceFoldersToMap ) [ 0 ] [ ] } )
259- remainingWorkspaceFoldersToMap = [ ]
260- for ( const [ prefix , folders ] of Object . entries ( workspacesByPrefixes ) ) {
261- // if a folder has a unique prefix
262- if ( folders . length === 1 && results [ prefix ] === undefined ) {
263- results [ prefix ] = folders [ 0 ] . folder
264- continue
265- }
266-
267- // find the folders that do not have more parents
268- const foldersToSuffix : typeof folders = [ ]
269- for ( const folder of folders ) {
270- if ( folder . preferredPrefixQueue . length > 0 ) {
271- remainingWorkspaceFoldersToMap . push ( folder )
272- } else {
273- foldersToSuffix . push ( folder )
274- }
275- }
276- // for these last resort folders, suffix them with an increasing number until unique
277- if ( foldersToSuffix . length === 1 && results [ prefix ] === undefined ) {
278- results [ prefix ] = foldersToSuffix [ 0 ] . folder
279- } else {
280- let suffix = 1
281- for ( const folder of foldersToSuffix ) {
282- let newPrefix : string
283- let safetyCounter = 0
284- do {
285- newPrefix = `${ prefix } ${ suffix }
286- suffix ++
287- safetyCounter ++
288- } while (
289- results [ newPrefix ] !== undefined &&
290- safetyCounter < workspaceFolderPrefixGuards . maximumFoldersWithMatchingSubfolders
291- )
292- if ( safetyCounter >= workspaceFolderPrefixGuards . maximumFoldersWithMatchingSubfolders ) {
293- throw new ToolkitError (
294- `Could not find a unique prefix for workspace folder ${ folder . folder . name }
295- )
296- }
297- results [ newPrefix ] = folder . folder
298- }
299- }
300- }
301- }
302- if ( remainingWorkspaceFoldersToMap . length > 0 ) {
303- throw new ToolkitError (
304- `Could not find a unique prefix for workspace folder ${ remainingWorkspaceFoldersToMap [ 0 ] . folder . name }
305- )
306- }
307-
308- return results
309- }
310-
31165/**
31266 * gets the absolute path from a zip path
31367 * @param zipFilePath the path in the zip file
0 commit comments