Skip to content

Commit 80d221d

Browse files
lmmxlmmx
committed
feat: allow user configuration of zstd compression level
1 parent 8007c1c commit 80d221d

File tree

2 files changed

+312
-8
lines changed

2 files changed

+312
-8
lines changed

‎packages/cache/__tests__/tar.test.ts

Lines changed: 280 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ test('zstd extract tar', async () => {
7373
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
7474
.concat([
7575
'--use-compress-program',
76-
IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30'
76+
IS_WINDOWS ? '"zstd -d -3 --long=30"' : 'unzstd --long=30'
7777
])
7878
.join(' '),
7979
undefined,
@@ -104,7 +104,7 @@ test('zstd extract tar with windows BSDtar', async () => {
104104
expect(execMock).toHaveBeenNthCalledWith(
105105
1,
106106
[
107-
'zstd -d --long=30 --force -o',
107+
'zstd -d -3 --long=30 --force -o',
108108
TarFilename.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
109109
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/')
110110
].join(' '),
@@ -233,7 +233,7 @@ test('zstd create tar', async () => {
233233
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
234234
.concat([
235235
'--use-compress-program',
236-
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
236+
IS_WINDOWS ? '"zstd -T0 -3 --long=30"' : 'zstdmt -3 --long=30'
237237
])
238238
.join(' '),
239239
undefined, // args
@@ -292,7 +292,7 @@ test('zstd create tar with windows BSDtar', async () => {
292292
expect(execMock).toHaveBeenNthCalledWith(
293293
2,
294294
[
295-
'zstd -T0 --long=30 --force -o',
295+
'zstd -T0 -3 --long=30 --force -o',
296296
CacheFilename.Zstd.replace(/\\/g, '/'),
297297
TarFilename.replace(/\\/g, '/')
298298
].join(' '),
@@ -480,3 +480,279 @@ test('gzip list tar', async () => {
480480
}
481481
)
482482
})
483+
484+
test('zstd create tar with custom compression level', async () => {
485+
const execMock = jest.spyOn(exec, 'exec')
486+
487+
// Save the original environment variable value
488+
const originalCompressionLevel = process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
489+
490+
try {
491+
// Set the custom compression level
492+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = '10'
493+
494+
const archiveFolder = getTempDir()
495+
const workspace = process.env['GITHUB_WORKSPACE']
496+
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
497+
498+
await fs.promises.mkdir(archiveFolder, {recursive: true})
499+
500+
await tar.createTar(
501+
archiveFolder,
502+
sourceDirectories,
503+
CompressionMethod.Zstd
504+
)
505+
506+
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
507+
508+
expect(execMock).toHaveBeenCalledTimes(1)
509+
expect(execMock).toHaveBeenCalledWith(
510+
[
511+
`"${tarPath}"`,
512+
'--posix',
513+
'-cf',
514+
IS_WINDOWS
515+
? CacheFilename.Zstd.replace(/\\/g, '/')
516+
: CacheFilename.Zstd,
517+
'--exclude',
518+
IS_WINDOWS
519+
? CacheFilename.Zstd.replace(/\\/g, '/')
520+
: CacheFilename.Zstd,
521+
'-P',
522+
'-C',
523+
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
524+
'--files-from',
525+
ManifestFilename
526+
]
527+
.concat(IS_WINDOWS ? ['--force-local'] : [])
528+
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
529+
.concat([
530+
'--use-compress-program',
531+
IS_WINDOWS ? '"zstd -T0 -10 --long=30"' : 'zstdmt -10 --long=30'
532+
])
533+
.join(' '),
534+
undefined, // args
535+
{
536+
cwd: archiveFolder,
537+
env: expect.objectContaining(defaultEnv)
538+
}
539+
)
540+
} finally {
541+
// Restore the original environment variable
542+
if (originalCompressionLevel === undefined) {
543+
delete process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
544+
} else {
545+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = originalCompressionLevel
546+
}
547+
}
548+
})
549+
550+
test('zstd create tar with invalid compression level', async () => {
551+
const execMock = jest.spyOn(exec, 'exec')
552+
553+
// Save the original environment variable value
554+
const originalCompressionLevel = process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
555+
556+
try {
557+
// Set an invalid compression level
558+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = 'invalid'
559+
560+
const archiveFolder = getTempDir()
561+
const workspace = process.env['GITHUB_WORKSPACE']
562+
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
563+
564+
await fs.promises.mkdir(archiveFolder, {recursive: true})
565+
566+
await tar.createTar(
567+
archiveFolder,
568+
sourceDirectories,
569+
CompressionMethod.Zstd
570+
)
571+
572+
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
573+
574+
// Should fall back to the default compression level (-3)
575+
expect(execMock).toHaveBeenCalledTimes(1)
576+
expect(execMock).toHaveBeenCalledWith(
577+
[
578+
`"${tarPath}"`,
579+
'--posix',
580+
'-cf',
581+
IS_WINDOWS
582+
? CacheFilename.Zstd.replace(/\\/g, '/')
583+
: CacheFilename.Zstd,
584+
'--exclude',
585+
IS_WINDOWS
586+
? CacheFilename.Zstd.replace(/\\/g, '/')
587+
: CacheFilename.Zstd,
588+
'-P',
589+
'-C',
590+
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
591+
'--files-from',
592+
ManifestFilename
593+
]
594+
.concat(IS_WINDOWS ? ['--force-local'] : [])
595+
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
596+
.concat([
597+
'--use-compress-program',
598+
IS_WINDOWS ? '"zstd -T0 -3 --long=30"' : 'zstdmt -3 --long=30'
599+
])
600+
.join(' '),
601+
undefined, // args
602+
{
603+
cwd: archiveFolder,
604+
env: expect.objectContaining(defaultEnv)
605+
}
606+
)
607+
} finally {
608+
// Restore the original environment variable
609+
if (originalCompressionLevel === undefined) {
610+
delete process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
611+
} else {
612+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = originalCompressionLevel
613+
}
614+
}
615+
})
616+
617+
test('zstd create tar with out-of-range compression level', async () => {
618+
const execMock = jest.spyOn(exec, 'exec')
619+
620+
// Save the original environment variable value
621+
const originalCompressionLevel = process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
622+
623+
try {
624+
// Set an out-of-range compression level
625+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = '25'
626+
627+
const archiveFolder = getTempDir()
628+
const workspace = process.env['GITHUB_WORKSPACE']
629+
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
630+
631+
await fs.promises.mkdir(archiveFolder, {recursive: true})
632+
633+
await tar.createTar(
634+
archiveFolder,
635+
sourceDirectories,
636+
CompressionMethod.Zstd
637+
)
638+
639+
const tarPath = IS_WINDOWS ? GnuTarPathOnWindows : defaultTarPath
640+
641+
// Should fall back to the default compression level (-3)
642+
expect(execMock).toHaveBeenCalledTimes(1)
643+
expect(execMock).toHaveBeenCalledWith(
644+
[
645+
`"${tarPath}"`,
646+
'--posix',
647+
'-cf',
648+
IS_WINDOWS
649+
? CacheFilename.Zstd.replace(/\\/g, '/')
650+
: CacheFilename.Zstd,
651+
'--exclude',
652+
IS_WINDOWS
653+
? CacheFilename.Zstd.replace(/\\/g, '/')
654+
: CacheFilename.Zstd,
655+
'-P',
656+
'-C',
657+
IS_WINDOWS ? workspace?.replace(/\\/g, '/') : workspace,
658+
'--files-from',
659+
ManifestFilename
660+
]
661+
.concat(IS_WINDOWS ? ['--force-local'] : [])
662+
.concat(IS_MAC ? ['--delay-directory-restore'] : [])
663+
.concat([
664+
'--use-compress-program',
665+
IS_WINDOWS ? '"zstd -T0 -3 --long=30"' : 'zstdmt -3 --long=30'
666+
])
667+
.join(' '),
668+
undefined, // args
669+
{
670+
cwd: archiveFolder,
671+
env: expect.objectContaining(defaultEnv)
672+
}
673+
)
674+
} finally {
675+
// Restore the original environment variable
676+
if (originalCompressionLevel === undefined) {
677+
delete process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
678+
} else {
679+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = originalCompressionLevel
680+
}
681+
}
682+
})
683+
684+
test('zstd create tar with windows BSDtar and custom compression level', async () => {
685+
if (IS_WINDOWS) {
686+
const execMock = jest.spyOn(exec, 'exec')
687+
jest
688+
.spyOn(utils, 'getGnuTarPathOnWindows')
689+
.mockReturnValue(Promise.resolve(''))
690+
691+
// Save the original environment variable value
692+
const originalCompressionLevel = process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
693+
694+
try {
695+
// Set a custom compression level
696+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = '19'
697+
698+
const archiveFolder = getTempDir()
699+
const workspace = process.env['GITHUB_WORKSPACE']
700+
const sourceDirectories = ['~/.npm/cache', `${workspace}/dist`]
701+
702+
await fs.promises.mkdir(archiveFolder, {recursive: true})
703+
704+
await tar.createTar(
705+
archiveFolder,
706+
sourceDirectories,
707+
CompressionMethod.Zstd
708+
)
709+
710+
const tarPath = SystemTarPathOnWindows
711+
712+
expect(execMock).toHaveBeenCalledTimes(2)
713+
714+
expect(execMock).toHaveBeenNthCalledWith(
715+
1,
716+
[
717+
`"${tarPath}"`,
718+
'--posix',
719+
'-cf',
720+
TarFilename.replace(/\\/g, '/'),
721+
'--exclude',
722+
TarFilename.replace(/\\/g, '/'),
723+
'-P',
724+
'-C',
725+
workspace?.replace(/\\/g, '/'),
726+
'--files-from',
727+
ManifestFilename
728+
].join(' '),
729+
undefined, // args
730+
{
731+
cwd: archiveFolder,
732+
env: expect.objectContaining(defaultEnv)
733+
}
734+
)
735+
736+
expect(execMock).toHaveBeenNthCalledWith(
737+
2,
738+
[
739+
'zstd -T0 -19 --long=30 --force -o',
740+
CacheFilename.Zstd.replace(/\\/g, '/'),
741+
TarFilename.replace(/\\/g, '/')
742+
].join(' '),
743+
undefined, // args
744+
{
745+
cwd: archiveFolder,
746+
env: expect.objectContaining(defaultEnv)
747+
}
748+
)
749+
} finally {
750+
// Restore the original environment variable
751+
if (originalCompressionLevel === undefined) {
752+
delete process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
753+
} else {
754+
process.env['CACHE_ZSTD_COMPRESSION_LEVEL'] = originalCompressionLevel
755+
}
756+
}
757+
}
758+
})

‎packages/cache/src/internal/tar.ts

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,20 @@ async function getDecompressionProgram(
204204
}
205205
}
206206

207+
// Get zstd compression level from env var (1-22), defaulting to 3 if invalid
208+
function getZstdCompressionLevel(): string {
209+
const defaultLevel = 3
210+
const levelEnv = process.env['CACHE_ZSTD_COMPRESSION_LEVEL']
211+
212+
// Check if it's a valid positive integer
213+
if (!levelEnv || !/^\d+$/.test(levelEnv)) {
214+
return `-${defaultLevel}`
215+
}
216+
217+
const level = parseInt(levelEnv, 10)
218+
return level >= 1 && level <= 22 ? `-${level}` : `-${defaultLevel}`
219+
}
220+
207221
// Used for creating the archive
208222
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
209223
// zstdmt is equivalent to 'zstd -T0'
@@ -219,26 +233,40 @@ async function getCompressionProgram(
219233
tarPath.type === ArchiveToolType.BSD &&
220234
compressionMethod !== CompressionMethod.Gzip &&
221235
IS_WINDOWS
236+
237+
// Get the compression level for zstd
238+
const compressionLevel =
239+
compressionMethod === CompressionMethod.Gzip
240+
? ''
241+
: getZstdCompressionLevel()
242+
222243
switch (compressionMethod) {
223244
case CompressionMethod.Zstd:
224245
return BSD_TAR_ZSTD
225246
? [
226-
'zstd -T0 --long=30 --force -o',
247+
`zstd -T0 ${compressionLevel} --long=30 --force -o`,
227248
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
228249
TarFilename
229250
]
230251
: [
231252
'--use-compress-program',
232-
IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30'
253+
IS_WINDOWS
254+
? `"zstd -T0 ${compressionLevel} --long=30"`
255+
: `zstdmt ${compressionLevel} --long=30`
233256
]
234257
case CompressionMethod.ZstdWithoutLong:
235258
return BSD_TAR_ZSTD
236259
? [
237-
'zstd -T0 --force -o',
260+
`zstd -T0 ${compressionLevel} --force -o`,
238261
cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
239262
TarFilename
240263
]
241-
: ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']
264+
: [
265+
'--use-compress-program',
266+
IS_WINDOWS
267+
? `"zstd -T0 ${compressionLevel}"`
268+
: `zstdmt ${compressionLevel}`
269+
]
242270
default:
243271
return ['-z']
244272
}

0 commit comments

Comments
 (0)