[pull] main from renovatebot:main

.github/workflows/build.yml

@@ -34,7 +34,7 @@ env:
   # are we on a release branch?
   DO_REALEASE: ${{ github.ref_name == github.event.repository.default_branch || github.ref_name == 'next' || startsWith(github.ref_name, 'maint/')}}
   NODE_VERSION: 22
-  PDM_VERSION: 2.24.0 # renovate: datasource=pypi depName=pdm
+  PDM_VERSION: 2.24.1 # renovate: datasource=pypi depName=pdm
   DRY_RUN: true
   TEST_LEGACY_DECRYPTION: true
   SPARSE_CHECKOUT: |-
@@ -605,7 +605,7 @@ jobs:
           os: ${{ runner.os }}
 
       - name: Setup PDM
-        uses: pdm-project/setup-pdm@deb8d8a4e2a03aabcef6f2cc981923fc6b29ef99 # v4.3
+        uses: pdm-project/setup-pdm@94a823180e06fcde4ad29308721954a521c96ed0 # v4.4
         with:
           python-version-file: .python-version
           version: ${{ env.PDM_VERSION }}

.nvmrc

@@ -1 +1 @@
-22.14.0
+22.15.0

lib/modules/datasource/npm/__snapshots__/index.spec.ts.snap

@@ -30,7 +30,6 @@ exports[`modules/datasource/npm/index > should fetch package info from custom re
 
 exports[`modules/datasource/npm/index > should fetch package info from npm 1`] = `
 {
-  "isPrivate": false,
   "registryUrl": "https://registry.npmjs.org",
   "releases": [
     {

lib/modules/datasource/npm/get.spec.ts

@@ -2,7 +2,8 @@ import { ExternalHostError } from '../../../types/errors/external-host-error';
 import * as _packageCache from '../../../util/cache/package';
 import * as hostRules from '../../../util/host-rules';
 import { Http } from '../../../util/http';
-import { CACHE_REVISION, getDependency } from './get';
+import type { HttpResponse } from '../../../util/http/types';
+import { getDependency } from './get';
 import { resolveRegistryUrl, setNpmrc } from './npmrc';
 import * as httpMock from '~test/http-mock';
 
@@ -563,69 +564,96 @@ describe('modules/datasource/npm/get', () => {
     `);
   });
 
-  it('discards cache with no revision', async () => {
-    setNpmrc('registry=https://test.org\n_authToken=XXX');
-
-    packageCache.get.mockResolvedValueOnce({
-      some: 'result',
-      cacheData: { softExpireAt: '2099' },
-    });
-
-    httpMock
-      .scope('https://test.org')
-      .get('/@neutrinojs%2Freact')
-      .reply(200, {
-        name: '@neutrinojs/react',
+  describe('cache', () => {
+    const httpResponse: HttpResponse<unknown> = {
+      statusCode: 200,
+      body: {
+        name: 'test',
+        repository: {
+          type: 'git',
+          url: 'https://github.com/octocat/Hello-World/tree/master/packages/test',
+          directory: 'packages/foo',
+        },
         versions: { '1.0.0': {} },
-      });
-    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
-    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
+        'dist-tags': { latest: '1.0.0' },
+      },
+      headers: { 'cache-control': 'max-age=180, public' },
+    };
 
-    expect(dep?.releases).toHaveLength(1);
-  });
+    it('returns unexpired cache', async () => {
+      vi.setSystemTime('2024-06-15T00:14:59.999Z');
+      packageCache.get.mockResolvedValue({
+        etag: 'some-etag',
+        timestamp: '2024-06-15T00:00:00.000Z',
+        httpResponse,
+      });
 
-  it('returns unexpired cache', async () => {
-    packageCache.get.mockResolvedValueOnce({
-      some: 'result',
-      cacheData: { revision: CACHE_REVISION, softExpireAt: '2099' },
+      const dep = await getDependency(
+        http,
+        'https://example.com',
+        'some-package',
+      );
+
+      expect(dep).toEqual({
+        registryUrl: 'https://example.com',
+        releases: [{ version: '1.0.0' }],
+        sourceDirectory: 'packages/foo',
+        sourceUrl:
+          'https://github.com/octocat/Hello-World/tree/master/packages/test',
+        tags: { latest: '1.0.0' },
+      });
     });
-    const dep = await getDependency(http, 'https://some.url', 'some-package');
-    expect(dep).toMatchObject({ some: 'result' });
-  });
 
-  it('returns soft expired cache if revalidated', async () => {
-    packageCache.get.mockResolvedValueOnce({
-      some: 'result',
-      cacheData: {
-        revision: CACHE_REVISION,
-        softExpireAt: '2020',
+    it('returns soft expired cache if revalidated', async () => {
+      vi.setSystemTime('2024-06-15T00:15:00.000Z');
+      packageCache.get.mockResolvedValue({
         etag: 'some-etag',
-      },
+        timestamp: '2024-06-15T00:00:00.000Z',
+        httpResponse,
+      });
+      setNpmrc('registry=https://example.com\n_authToken=XXX');
+      httpMock.scope('https://example.com').get('/some-package').reply(304);
+
+      const dep = await getDependency(
+        http,
+        'https://example.com',
+        'some-package',
+      );
+
+      expect(dep).toEqual({
+        registryUrl: 'https://example.com',
+        releases: [{ version: '1.0.0' }],
+        sourceDirectory: 'packages/foo',
+        sourceUrl:
+          'https://github.com/octocat/Hello-World/tree/master/packages/test',
+        tags: { latest: '1.0.0' },
+      });
     });
-    setNpmrc('registry=https://test.org\n_authToken=XXX');
-
-    httpMock.scope('https://test.org').get('/@neutrinojs%2Freact').reply(304);
-    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
-    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
-    expect(dep).toMatchObject({ some: 'result' });
-  });
 
-  it('returns soft expired cache on npmjs error', async () => {
-    packageCache.get.mockResolvedValueOnce({
-      some: 'result',
-      cacheData: {
-        revision: CACHE_REVISION,
-        softExpireAt: '2020',
+    it('returns soft expired cache on npmjs error', async () => {
+      vi.setSystemTime('2024-06-15T00:15:00.000Z');
+      packageCache.get.mockResolvedValue({
         etag: 'some-etag',
-      },
+        timestamp: '2024-06-15T00:00:00.000Z',
+        httpResponse,
+      });
+      setNpmrc('registry=https://example.com\n_authToken=XXX');
+      httpMock.scope('https://example.com').get('/some-package').reply(500);
+
+      const dep = await getDependency(
+        http,
+        'https://example.com',
+        'some-package',
+      );
+
+      expect(dep).toEqual({
+        registryUrl: 'https://example.com',
+        releases: [{ version: '1.0.0' }],
+        sourceDirectory: 'packages/foo',
+        sourceUrl:
+          'https://github.com/octocat/Hello-World/tree/master/packages/test',
+        tags: { latest: '1.0.0' },
+      });
     });
-
-    httpMock
-      .scope('https://registry.npmjs.org')
-      .get('/@neutrinojs%2Freact')
-      .reply(500);
-    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
-    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
-    expect(dep).toMatchObject({ some: 'result' });
   });
 });

lib/modules/datasource/npm/get.ts

@@ -1,23 +1,17 @@
-import url from 'node:url';
 import is from '@sindresorhus/is';
-import { DateTime } from 'luxon';
 import { z } from 'zod';
-import { GlobalConfig } from '../../../config/global';
 import { HOST_DISABLED } from '../../../constants/error-messages';
 import { logger } from '../../../logger';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
-import * as packageCache from '../../../util/cache/package';
 import * as hostRules from '../../../util/host-rules';
 import type { Http } from '../../../util/http';
+import { PackageHttpCacheProvider } from '../../../util/http/cache/package-http-cache-provider';
 import type { HttpOptions } from '../../../util/http/types';
 import { regEx } from '../../../util/regex';
-import { HttpCacheStats } from '../../../util/stats';
 import { asTimestamp } from '../../../util/timestamp';
 import { joinUrlParts } from '../../../util/url';
 import type { Release, ReleaseResult } from '../types';
-import type { CachedReleaseResult, NpmResponse } from './types';
-
-export const CACHE_REVISION = 1;
+import type { NpmResponse } from './types';
 
 const SHORT_REPO_REGEX = regEx(
   /^((?<platform>bitbucket|github|gitlab):)?(?<shortRepo>[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)$/,
@@ -81,53 +75,12 @@ export async function getDependency(
 
   const packageUrl = joinUrlParts(registryUrl, packageName.replace('/', '%2F'));
 
-  // Now check the persistent cache
-  const cacheNamespace = 'datasource-npm:data';
-  const cachedResult = await packageCache.get<CachedReleaseResult>(
-    cacheNamespace,
-    packageUrl,
-  );
-  if (cachedResult?.cacheData) {
-    if (cachedResult.cacheData.revision === CACHE_REVISION) {
-      const softExpireAt = DateTime.fromISO(
-        cachedResult.cacheData.softExpireAt,
-      );
-      if (softExpireAt.isValid && softExpireAt > DateTime.local()) {
-        logger.trace('Cached result is not expired - reusing');
-        HttpCacheStats.incLocalHits(packageUrl);
-        delete cachedResult.cacheData;
-        return cachedResult;
-      }
-
-      logger.trace('Cached result is soft expired');
-      HttpCacheStats.incLocalMisses(packageUrl);
-    } else {
-      logger.trace(
-        `Package cache for npm package "${packageName}" is from an old revision - discarding`,
-      );
-      delete cachedResult.cacheData;
-    }
-  }
-  const cacheMinutes = 15;
-  const softExpireAt = DateTime.local().plus({ minutes: cacheMinutes }).toISO();
-  let cacheHardTtlMinutes = GlobalConfig.get('cacheHardTtlMinutes');
-  if (
-    !(
-      is.number(cacheHardTtlMinutes) &&
-      /* istanbul ignore next: needs test */ cacheHardTtlMinutes > cacheMinutes
-    )
-  ) {
-    cacheHardTtlMinutes = cacheMinutes;
-  }
-
-  const uri = url.parse(packageUrl);
-
   try {
-    const options: HttpOptions = {};
-    if (cachedResult?.cacheData?.etag) {
-      logger.trace({ packageName }, 'Using cached etag');
-      options.headers = { 'If-None-Match': cachedResult.cacheData.etag };
-    }
+    const cacheProvider = new PackageHttpCacheProvider({
+      namespace: 'datasource-npm:cache-provider',
+      checkAuthorizationHeader: false,
+    });
+    const options: HttpOptions = { cacheProvider };
 
     // set abortOnError for registry.npmjs.org if no hostRule with explicit abortOnError exists
     if (
@@ -145,23 +98,8 @@ export async function getDependency(
       });
     }
 
-    const raw = await http.getJsonUnchecked<NpmResponse>(packageUrl, options);
-    if (cachedResult?.cacheData && raw.statusCode === 304) {
-      logger.trace(`Cached npm result for ${packageName} is revalidated`);
-      HttpCacheStats.incRemoteHits(packageUrl);
-      cachedResult.cacheData.softExpireAt = softExpireAt;
-      await packageCache.set(
-        cacheNamespace,
-        packageUrl,
-        cachedResult,
-        cacheHardTtlMinutes,
-      );
-      delete cachedResult.cacheData;
-      return cachedResult;
-    }
-    HttpCacheStats.incRemoteMisses(packageUrl);
-    const etag = raw.headers.etag;
-    const res = raw.body;
+    const resp = await http.getJsonUnchecked<NpmResponse>(packageUrl, options);
+    const { body: res } = resp;
     if (!res.versions || !Object.keys(res.versions).length) {
       // Registry returned a 200 OK but with no versions
       logger.debug(`No versions returned for npm dependency ${packageName}`);
@@ -226,25 +164,16 @@ export async function getDependency(
       }
       return release;
     });
-    logger.trace({ dep }, 'dep');
-    const cacheControl = raw.headers?.['cache-control'];
-    if (
-      is.nonEmptyString(cacheControl) &&
-      regEx(/(^|,)\s*public\s*(,|$)/).test(cacheControl)
-    ) {
-      dep.isPrivate = false;
-      const cacheData = { revision: CACHE_REVISION, softExpireAt, etag };
-      await packageCache.set(
-        cacheNamespace,
-        packageUrl,
-        { ...dep, cacheData },
-        etag
-          ? /* istanbul ignore next: needs test */ cacheHardTtlMinutes
-          : cacheMinutes,
-      );
-    } else {
+
+    const isPublic = resp.headers?.['cache-control']
+      ?.toLocaleLowerCase()
+      ?.split(regEx(/\s*,\s*/))
+      ?.includes('public');
+    if (!isPublic) {
       dep.isPrivate = true;
     }
+
+    logger.trace({ dep }, 'dep');
     return dep;
   } catch (err) {
     const actualError = err instanceof ExternalHostError ? err.err : err;
@@ -259,15 +188,6 @@ export async function getDependency(
     }
 
     if (err instanceof ExternalHostError) {
-      if (cachedResult) {
-        logger.warn(
-          { err, host: uri.host },
-          `npm host error, reusing expired cached result instead`,
-        );
-        delete cachedResult.cacheData;
-        return cachedResult;
-      }
-
       if (actualError.name === 'ParseError' && actualError.body) {
         actualError.body = 'err.body deleted by Renovate';
         err.err = actualError;

lib/modules/datasource/npm/index.spec.ts

@@ -55,10 +55,10 @@ describe('modules/datasource/npm/index', () => {
     httpMock
       .scope('https://registry.npmjs.org')
       .get('/foobar')
-      .reply(200, npmResponse, { 'Cache-control': 'public, expires=300' });
+      .reply(200, npmResponse, { 'Cache-Control': 'public, expires=300' });
     const res = await getPkgReleases({ datasource, packageName: 'foobar' });
     expect(res).toMatchSnapshot();
-    expect(res?.isPrivate).toBeFalse();
+    expect(res?.isPrivate).toBeUndefined();
   });
 
   it('should parse repo url', async () => {

lib/modules/datasource/npm/types.ts

@@ -1,6 +1,5 @@
 import type { PackageRule } from '../../../config/types';
 import type { HostRule } from '../../../types';
-import type { ReleaseResult } from '../types';
 
 export interface NpmrcRules {
   hostRules: HostRule[];
@@ -33,12 +32,4 @@ export interface NpmResponse {
   'dist-tags'?: Record<string, string>;
 }
 
-export interface CachedReleaseResult extends ReleaseResult {
-  cacheData?: {
-    revision?: number;
-    etag: string | undefined;
-    softExpireAt: string;
-  };
-}
-
 export type Npmrc = Record<string, any>;

lib/modules/manager/api.ts

@@ -85,7 +85,6 @@ import * as preCommit from './pre-commit';
 import * as pub from './pub';
 import * as puppet from './puppet';
 import * as pyenv from './pyenv';
-import * as renovateConfigPresets from './renovate-config-presets';
 import * as rubyVersion from './ruby-version';
 import * as runtimeVersion from './runtime-version';
 import * as sbt from './sbt';
@@ -195,7 +194,6 @@ api.set('pre-commit', preCommit);
 api.set('pub', pub);
 api.set('puppet', puppet);
 api.set('pyenv', pyenv);
-api.set('renovate-config-presets', renovateConfigPresets);
 api.set('ruby-version', rubyVersion);
 api.set('runtime-version', runtimeVersion);
 api.set('sbt', sbt);