diff --git a/lib/src/command/cache.dart b/lib/src/command/cache.dart index 4a370f7287..8ec1f92d44 100644 --- a/lib/src/command/cache.dart +++ b/lib/src/command/cache.dart @@ -5,6 +5,7 @@ import '../command.dart'; import 'cache_add.dart'; import 'cache_clean.dart'; +import 'cache_gc.dart'; import 'cache_list.dart'; import 'cache_preload.dart'; import 'cache_repair.dart'; @@ -24,5 +25,6 @@ class CacheCommand extends PubCommand { addSubcommand(CacheCleanCommand()); addSubcommand(CacheRepairCommand()); addSubcommand(CachePreloadCommand()); + addSubcommand(CacheGcCommand()); } } diff --git a/lib/src/command/cache_gc.dart b/lib/src/command/cache_gc.dart new file mode 100644 index 0000000000..64b70fd9f6 --- /dev/null +++ b/lib/src/command/cache_gc.dart @@ -0,0 +1,139 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; + +import '../command.dart'; +import '../command_runner.dart'; +import '../io.dart'; +import '../log.dart' as log; +import '../package_config.dart'; +import '../utils.dart'; + +class CacheGcCommand extends PubCommand { + @override + String get name => 'gc'; + @override + String get description => 'Prunes unused packages from the system cache.'; + @override + bool get takesArguments => false; + + final dontRemoveFilesOlderThan = + runningFromTest ? const Duration(seconds: 2) : const Duration(hours: 2); + + CacheGcCommand() { + argParser.addFlag( + 'force', + abbr: 'f', + help: 'Prune cache without confirmation', + ); + } + + @override + Future runProtected() async { + final activeRoots = cache.activeRoots(); + final validActiveRoots = []; + final paths = {}; + for (final packageConfigPath in activeRoots) { + late final PackageConfig packageConfig; + try { + packageConfig = PackageConfig.fromJson( + json.decode(readTextFile(packageConfigPath)), + ); + } on IOException catch (e) { + // Failed to read file - probably got deleted. + log.fine('Failed to read packageConfig $packageConfigPath: $e'); + continue; + } on FormatException catch (e) { + log.warning( + 'Failed to decode packageConfig $packageConfigPath: $e.\n' + 'It could be corrupted', + ); + // Failed to decode - probably corrupted. + continue; + } + for (final package in packageConfig.packages) { + final rootUri = package.resolvedRootDir(packageConfigPath); + if (p.isWithin(cache.rootDir, rootUri)) { + paths.add(rootUri); + } + } + validActiveRoots.add(packageConfigPath); + } + final now = DateTime.now(); + final allPathsToGC = + [ + for (final source in cache.cachedSources) + ...await source.entriesToGc( + cache, + paths + .where( + (path) => p.isWithin( + p.canonicalize(cache.rootDirForSource(source)), + path, + ), + ) + .toSet(), + ), + ].where((path) { + // Only clear cache entries older than 2 hours to avoid race + // conditions with ongoing `pub get` processes. + final s = statPath(path); + if (s.type == FileSystemEntityType.notFound) return false; + return now.difference(s.modified) > dontRemoveFilesOlderThan; + }).toList(); + if (validActiveRoots.isEmpty) { + log.message('Found no active projects.'); + } else { + final s = validActiveRoots.length == 1 ? '' : 's'; + log.message('Found ${validActiveRoots.length} active project$s:'); + for (final packageConfigPath in validActiveRoots) { + final parts = p.split(packageConfigPath); + var projectDir = packageConfigPath; + if (parts[parts.length - 2] == '.dart_tool' && + parts[parts.length - 1] == 'package_config.json') { + projectDir = p.joinAll(parts.sublist(0, parts.length - 2)); + } + log.message('* $projectDir'); + } + } + var sum = 0; + for (final entry in allPathsToGC) { + if (dirExists(entry)) { + for (final file in listDir( + entry, + recursive: true, + includeHidden: true, + includeDirs: false, + )) { + sum += tryStatFile(file)?.size ?? 0; + } + } else { + sum += tryStatFile(entry)?.size ?? 0; + } + } + if (sum == 0) { + log.message('No unused cache entries found.'); + return; + } + log.message(''); + log.message( + ''' +All other projects will need to run `$topLevelProgram pub get` again to work correctly.''', + ); + log.message('Will recover ${readableFileSize(sum)}.'); + + if (argResults.flag('force') || + await confirm('Are you sure you want to continue?')) { + await log.progress('Deleting unused cache entries', () async { + for (final path in allPathsToGC..sort()) { + tryDeleteEntry(path); + } + }); + } + } +} diff --git a/lib/src/command/lish.dart b/lib/src/command/lish.dart index a2d405f87f..c4776aab86 100644 --- a/lib/src/command/lish.dart +++ b/lib/src/command/lish.dart @@ -359,7 +359,7 @@ the \$PUB_HOSTED_URL environment variable.'''); baseDir: entrypoint.workPackage.dir, ).toBytes(); - final size = _readableFileSize(packageBytes.length); + final size = readableFileSize(packageBytes.length); log.message('\nTotal compressed archive size: $size.\n'); final validationResult = @@ -526,18 +526,6 @@ the \$PUB_HOSTED_URL environment variable.'''); } } -String _readableFileSize(int size) { - if (size >= 1 << 30) { - return '${size ~/ (1 << 30)} GB'; - } else if (size >= 1 << 20) { - return '${size ~/ (1 << 20)} MB'; - } else if (size >= 1 << 10) { - return '${size ~/ (1 << 10)} KB'; - } else { - return '<1 KB'; - } -} - class _Publication { Uint8List packageBytes; int warningCount; diff --git a/lib/src/source/cached.dart b/lib/src/source/cached.dart index 2cfa2a1ffa..67c4e1bc25 100644 --- a/lib/src/source/cached.dart +++ b/lib/src/source/cached.dart @@ -76,6 +76,14 @@ abstract class CachedSource extends Source { /// Returns a list of results indicating for each if that package was /// successfully repaired. Future> repairCachedPackages(SystemCache cache); + + /// Return all files directories inside this source that can be removed while + /// preserving the packages given by [alivePackages] a list of package root + /// directories. They should all be canonicalized. + Future> entriesToGc( + SystemCache cache, + Set alivePackages, + ); } /// The result of repairing a single cache entry. diff --git a/lib/src/source/git.dart b/lib/src/source/git.dart index 9087944659..f6ed7b09cf 100644 --- a/lib/src/source/git.dart +++ b/lib/src/source/git.dart @@ -900,6 +900,74 @@ class GitSource extends CachedSource { } return name; } + + @override + Future> entriesToGc( + SystemCache cache, + Set alivePackages, + ) async { + final rootDir = p.canonicalize(cache.rootDirForSource(this)); + if (!entryExists(rootDir)) return const []; + + final gitDirsToRemove = {}; + // First enumerate all git repos inside [rootDir]. + for (final entry in listDir(rootDir)) { + final gitEntry = p.join(entry, '.git'); + if (!entryExists(gitEntry)) continue; + gitDirsToRemove.add(p.canonicalize(entry)); + } + final cacheDirsToRemove = {}; + try { + cacheDirsToRemove.addAll( + listDir(p.join(rootDir, 'cache')).map(p.canonicalize), + ); + } on IOException { + // Most likely the directory didn't exist. + // ignore. + } + // For each package walk up parent directories to find the containing git + // repo, and mark it alive by removing from `gitDirsToRemove`. + for (final alivePackage in alivePackages) { + var candidate = p.canonicalize(alivePackage); + while (!p.equals(candidate, rootDir)) { + if (gitDirsToRemove.remove(candidate)) { + // Package is alive, now also retain its cachedir. + // + // TODO(sigurdm): Should we just GC all cache-dirs? They are not + // needed for consuming packages, and most likely will be recreated + // when needed. + final gitEntry = p.join(candidate, '.git'); + try { + if (dirExists(gitEntry)) { + final path = + (await git.run([ + 'remote', + 'get-url', + 'origin', + ], workingDir: candidate)).split('\n').first; + cacheDirsToRemove.remove(p.canonicalize(path)); + } else if (fileExists(gitEntry)) { + // Potential future - using worktrees. + final path = + (await git.run([ + 'worktree', + 'list', + '--porcelain', + ], workingDir: candidate)).split('\n').first.split(' ').last; + cacheDirsToRemove.remove(p.canonicalize(path)); + } + } on git.GitException catch (e) { + log.fine('Failed to find canonical cache for $candidate, $e'); + } + break; + } + // Try the parent directory + candidate = p.dirname(candidate); + } + } + + return [...cacheDirsToRemove, ...gitDirsToRemove]; + } } class GitDescription extends Description { diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index 269c7a0dbb..8114f5fd0c 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart @@ -1761,6 +1761,62 @@ See $contentHashesDocumentationUrl. } } + @override + Future> entriesToGc( + SystemCache cache, + Set alivePackages, + ) async { + final root = p.canonicalize(cache.rootDirForSource(this)); + final result = {}; + final List hostDirs; + + try { + hostDirs = listDir(root); + } on IOException { + // Hosted cache seems uninitialized. GC nothing. + return []; + } + for (final hostDir in hostDirs) { + final List packageDirs; + try { + packageDirs = listDir(hostDir).map(p.canonicalize).toList(); + } on IOException { + // Failed to list `hostDir`. Perhaps a stray file? Skip. + continue; + } + for (final packageDir in packageDirs) { + if (!alivePackages.contains(packageDir)) { + result.add(packageDir); + // Also clear the associated hash file. + final hashFile = p.join( + cache.rootDir, + 'hosted-hashes', + p.basename(hostDir), + '${p.basename(packageDir)}.sha256', + ); + if (fileExists(hashFile)) { + result.add(hashFile); + } + } + } + // Clear all version listings older than two days, they'd likely need to + // be re-fetched anyways: + for (final cacheFile in listDir( + p.join(hostDir, _versionListingDirectory), + )) { + final stat = tryStatFile(cacheFile); + + if (stat != null && + DateTime.now().difference(stat.modified) > + const Duration(days: 2)) { + result.add(cacheFile); + } + } + } + + return result.toList(); + } + /// Enables speculative prefetching of dependencies of packages queried with /// [doGetVersions]. Future withPrefetching(Future Function() callback) async { diff --git a/lib/src/utils.dart b/lib/src/utils.dart index 2a62a94071..a769cf6019 100644 --- a/lib/src/utils.dart +++ b/lib/src/utils.dart @@ -825,3 +825,15 @@ extension ExpectEntries on YamlList { ), ]; } + +String readableFileSize(int size) { + if (size >= 1 << 30) { + return '${size ~/ (1 << 30)} GB'; + } else if (size >= 1 << 20) { + return '${size ~/ (1 << 20)} MB'; + } else if (size >= 1 << 10) { + return '${size ~/ (1 << 10)} KB'; + } else { + return '<1 KB'; + } +} diff --git a/pubspec.yaml b/pubspec.yaml index 38570c1602..27b9267e1f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -18,7 +18,7 @@ dependencies: http_parser: ^4.1.2 meta: ^1.17.0 path: ^1.9.1 - pool: ^1.5.1 + pool: ^1.0.0 pub_semver: ^2.2.0 shelf: ^1.4.2 source_span: ^1.10.1 diff --git a/test/cache/gc_test.dart b/test/cache/gc_test.dart index a56944e845..280b601036 100644 --- a/test/cache/gc_test.dart +++ b/test/cache/gc_test.dart @@ -90,4 +90,112 @@ void main() async { ), }); }); + + test('gcing and empty cache behaves well', () async { + await runPub( + args: ['cache', 'gc', '--force'], + output: allOf( + contains('Found no active projects.'), + contains('No unused cache entries found.'), + ), + ); + }); + + test('Can gc cache entries', () async { + final server = await servePackages(); + + server.serve('hosted1', '1.0.0'); + server.serve('hosted2', '1.0.0'); + + await d.git('git1', [d.libPubspec('git1', '1.0.0')]).create(); + await d.git('git2', [d.libPubspec('git2', '1.0.0')]).create(); + + await d.git('git_with_path1', [ + d.dir('pkg', [d.libPubspec('git_with_path1', '1.0.0')]), + ]).create(); + await d.git('git_with_path2', [ + d.dir('pkg', [d.libPubspec('git_with_path2', '1.0.0')]), + ]).create(); + + await d + .appDir( + dependencies: { + 'hosted1': '1.0.0', + 'git1': {'git': '../git1'}, + 'git_with_path1': { + 'git': {'url': '../git_with_path1', 'path': 'pkg'}, + }, + }, + ) + .create(); + await pubGet(); + await d + .appDir( + dependencies: { + 'hosted2': '1.0.0', + 'git2': {'git': '../git2'}, + 'git_with_path2': { + 'git': {'url': '../git_with_path2', 'path': 'pkg'}, + }, + }, + ) + .create(); + await pubGet(output: contains('- hosted1')); + + await runPub( + args: ['cache', 'gc', '--force'], + output: allOf( + contains('* ${p.join(d.sandbox, appPath).toLowerCase()}'), + contains('No unused cache entries found'), + ), + ); + await Future.delayed(const Duration(seconds: 2)); + + await runPub( + args: ['cache', 'gc', '--force'], + output: allOf( + contains('* ${p.join(d.sandbox, appPath).toLowerCase()}'), + contains(RegExp('Will recover [0-9]{3} KB.')), + ), + silent: allOf([ + contains(RegExp('Deleting directory .*git.*cache/git1-.*')), + contains(RegExp('Deleting directory .*git.*cache/git_with_path1-.*')), + contains(RegExp('Deleting directory .*git.*git1-.*')), + contains(RegExp('Deleting directory .*git.*git_with_path1-.*')), + contains( + RegExp('Deleting file .*hosted-hashes.*hosted1-1.0.0.sha256.'), + ), + contains(RegExp('Deleting directory .*hosted.*hosted1-1.0.0.')), + isNot(contains(RegExp('Deleting.*hosted2'))), + isNot(contains(RegExp('Deleting.*git2'))), + isNot(contains(RegExp('Deleting.*git_with_path2'))), + ]), + ); + expect( + Directory( + p.join(d.sandbox, d.hostedCachePath(), 'hosted1-1.0.0'), + ).existsSync(), + isFalse, + ); + expect( + Directory( + p.join(d.sandbox, d.hostedCachePath(), 'hosted2-1.0.0'), + ).existsSync(), + isTrue, + ); + + expect( + Directory( + p.join(d.sandbox, cachePath, 'git'), + ).listSync().map((f) => p.basename(f.path)), + {'cache', matches(RegExp('git2.*')), matches(RegExp('git_with_path2.*'))}, + ); + + expect( + Directory( + p.join(d.sandbox, cachePath, 'git', 'cache'), + ).listSync().map((f) => p.basename(f.path)), + {matches(RegExp('git2.*')), matches(RegExp('git_with_path2.*'))}, + ); + }); } diff --git a/test/testdata/goldens/help_test/pub cache --help.txt b/test/testdata/goldens/help_test/pub cache --help.txt index fdb0b2c09b..d9a64276eb 100644 --- a/test/testdata/goldens/help_test/pub cache --help.txt +++ b/test/testdata/goldens/help_test/pub cache --help.txt @@ -10,6 +10,7 @@ Usage: pub cache [arguments...] Available subcommands: add Install a package. clean Clears the global PUB_CACHE. + gc Prunes unused packages from the system cache. repair Reinstall cached packages. Run "pub help" to see global options.