[Fleet] Improve installation of bundled packages in airgapped environments (#230992)

Closes #224954

## Summary

Improve installation of bundled packages in airgapped environments.

- Add a fallback fetch of bundled packages / cached packages when
possible
- Improve error handling when EPR is not available
- Fetch bundled packages as fallback when is possible (regardless of EPR
being available or not)

Approach explained
[here](#224954 (comment))

### Testing steps

Configuration in `kibana.dev.yml`:
```
xpack.fleet:
    isAirGapped: true
    registryUrl: http://not-reachable
    developer.bundledPackageLocation: "/Users/<USERNAME>/Dev/kibana/x-pack/platform/plugins/shared/fleet/target/bundled_packages"
```
Bundled packages (APM, synthetics, etc) need to be present in the above
folder. Download them from EPR

- Put an older version of APM (for instance 7.16) in the folder and
install it going to `app/integrations/apm-7.16/overview`, add it to a
new agent policy. I tested with a policy that doesn't have `system`
installed
- temporarily remove the configs `isAirGapped` and `registryUrl` to be
able to reach the registry and add a newer version of the package to
`bundled_packages` folder (I used apm-8.2.0)
- To trigger the package upgrade, go to
`app/integrations/apm-8.2.0/overview`
- Put back the configs in `kibana.dev.yml` and now upgrade the package
and the policies
-  The upgrade should work properly and the policy revision is aligned
<img width="1862" height="965" alt="Screenshot 2025-08-11 at 11 13 29"
src="https://github.com/user-attachments/assets/690068f9-e43e-4685-b341-b3e4139e421e"
/>
<img width="1866" height="1159" alt="Screenshot 2025-08-11 at 11 13 58"
src="https://github.com/user-attachments/assets/9a12e223-6215-4759-ab89-eb554d6bc63e"
/>

**NOTE**: When I tested with a policy that has also other packages
installed (like system), the policy recompile still fails with a
connection error. I'm not quite sure how to solve this, since Fleet
needs to reach EPR to fetch the packages that are not bundled.


### Checklist

Check the PR satisfies following conditions. 

- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Elastic Machine <[email protected]>

Author: criamico

x-pack/platform/plugins/shared/fleet/server/services/epm/registry/index.test.ts

@@ -4,10 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
 import { loggingSystemMock } from '@kbn/core-logging-server-mocks';
 
-import { PackageNotFoundError, RegistryResponseError } from '../../../errors';
+import {
+  FleetError,
+  PackageNotFoundError,
+  RegistryConnectionError,
+  RegistryResponseError,
+} from '../../../errors';
 import * as Archive from '../archive';
 
 import {
@@ -18,18 +22,26 @@ import {
   getLicensePath,
   fetchCategories,
   fetchList,
+  getPackage,
 } from '.';
 
-const mockLoggerFactory = loggingSystemMock.create();
-const mockLogger = mockLoggerFactory.get('mock logger');
+const mockLogger = loggingSystemMock.create().get();
+
 const mockGetConfig = jest.fn();
 
 const mockGetBundledPackageByName = jest.fn();
 const mockFetchUrl = jest.fn();
+const mockGetResponseStreamWithSize = jest.fn();
+const mockStreamToBuffer = jest.fn();
+const mockVerifyPackageArchiveSignature = jest.fn();
+const mockGetPackageAssetsMapCache = jest.fn();
 
 const MockArchive = Archive as jest.Mocked<typeof Archive>;
 
 jest.mock('../archive');
+jest.mock('./requests');
+jest.mock('../streams');
+jest.mock('../packages/cache');
 
 jest.mock('../..', () => ({
   appContextService: {
@@ -43,12 +55,31 @@ jest.mock('../..', () => ({
 
 jest.mock('./requests', () => ({
   fetchUrl: (url: string) => mockFetchUrl(url),
+  getResponseStreamWithSize: (url: string) => mockGetResponseStreamWithSize(url),
+}));
+
+jest.mock('../streams', () => ({
+  streamToBuffer: (stream: NodeJS.ReadableStream, size?: number) =>
+    mockStreamToBuffer(stream, size),
 }));
 
 jest.mock('../packages/bundled_packages', () => ({
   getBundledPackageByName: (name: string) => mockGetBundledPackageByName(name),
 }));
 
+jest.mock('../packages/package_verification', () => ({
+  verifyPackageArchiveSignature: (
+    pkgName: string,
+    pkgVersion: string,
+    pkgArchiveBuffer: Buffer | undefined,
+    logger: Logger
+  ) => mockVerifyPackageArchiveSignature(pkgName, pkgVersion, pkgArchiveBuffer, logger),
+}));
+
+jest.mock('../packages/cache', () => ({
+  getPackageAssetsMapCache: () => mockGetPackageAssetsMapCache(),
+}));
+
 describe('splitPkgKey', () => {
   it('throws an error if there is nothing before the delimiter', () => {
     expect(() => {
@@ -225,6 +256,15 @@ describe('fetchInfo', () => {
     });
   });
 
+  it('falls back to bundled package when isAirGapped config == true', async () => {
+    mockGetConfig.mockReturnValue({
+      isAirGapped: true,
+    });
+
+    const fetchedInfo = await fetchInfo('test-package', '1.0.0');
+    expect(fetchedInfo).toBeTruthy();
+  });
+
   it('falls back to bundled package when one exists', async () => {
     const fetchedInfo = await fetchInfo('test-package', '1.0.0');
     expect(fetchedInfo).toBeTruthy();
@@ -237,15 +277,6 @@ describe('fetchInfo', () => {
       expect(e).toBeInstanceOf(PackageNotFoundError);
     }
   });
-
-  it('falls back to bundled package when isAirGapped config == true', async () => {
-    mockGetConfig.mockReturnValue({
-      isAirGapped: true,
-    });
-
-    const fetchedInfo = await fetchInfo('test-package', '1.0.0');
-    expect(fetchedInfo).toBeTruthy();
-  });
 });
 
 describe('fetchCategories', () => {
@@ -378,3 +409,200 @@ describe('fetchList', () => {
     expect(callUrl.searchParams.get('kibana.version')).toBeNull();
   });
 });
+
+describe('getPackage', () => {
+  const bundledPackage = {
+    name: 'testpkg',
+    version: '1.0.0',
+    getBuffer: () => Promise.resolve(Buffer.from('testpkg')),
+  };
+  const registryPackage = {
+    name: 'testpkg',
+    version: '1.0.1',
+    getBuffer: () => Promise.resolve(Buffer.from('testpkg')),
+  };
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  it('should return bundled package if isAirGapped = true', async () => {
+    mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage]));
+    mockGetResponseStreamWithSize.mockResolvedValue({ stream: {}, size: 1000 });
+    mockStreamToBuffer.mockResolvedValue(Buffer.from('testpkg'));
+    mockVerifyPackageArchiveSignature.mockResolvedValue('verified');
+    mockGetConfig.mockReturnValue({
+      isAirGapped: true,
+      enabled: true,
+      agents: { enabled: true, elasticsearch: {} },
+    });
+    MockArchive.unpackBufferToAssetsMap.mockReturnValue({
+      assetsMap: new Map(),
+      paths: [],
+      archiveIterator: {},
+    } as any);
+    MockArchive.generatePackageInfoFromArchiveBuffer.mockReturnValue({
+      packageInfo: { name: 'testpkg', version: '1.0.0' },
+    } as any);
+
+    mockGetBundledPackageByName.mockResolvedValue(bundledPackage);
+    const result = await getPackage('testpkg', '1.0.1');
+    expect(result).toEqual({
+      archiveIterator: {},
+      assetsMap: new Map(),
+      packageInfo: {
+        name: 'testpkg',
+        version: '1.0.0',
+      },
+      paths: [],
+      verificationResult: 'verified',
+    });
+  });
+
+  it('should return registry package', async () => {
+    mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage]));
+    mockGetResponseStreamWithSize.mockResolvedValue({ stream: {}, size: 1000 });
+    mockStreamToBuffer.mockResolvedValue(Buffer.from('testpkg'));
+    mockVerifyPackageArchiveSignature.mockResolvedValue('verified');
+    MockArchive.unpackBufferToAssetsMap.mockReturnValue({
+      assetsMap: new Map(),
+      paths: [],
+      archiveIterator: {},
+    } as any);
+    MockArchive.generatePackageInfoFromArchiveBuffer.mockReturnValue({
+      packageInfo: { name: 'testpkg', version: '1.0.1' },
+    } as any);
+
+    mockGetBundledPackageByName.mockResolvedValue(undefined);
+    const result = await getPackage('testpkg', '1.0.1');
+    expect(result).toEqual({
+      archiveIterator: {},
+      assetsMap: new Map(),
+      packageInfo: {
+        name: 'testpkg',
+        version: '1.0.1',
+      },
+      paths: [],
+      verificationResult: 'verified',
+    });
+  });
+
+  it('should throw if there is an error in fetchArchiveBuffer', async () => {
+    mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage]));
+    mockGetResponseStreamWithSize.mockRejectedValueOnce(new FleetError('Error fetching package'));
+    mockStreamToBuffer.mockResolvedValue(Buffer.from('testpkg'));
+    mockVerifyPackageArchiveSignature.mockResolvedValue('verified');
+    MockArchive.unpackBufferToAssetsMap.mockReturnValue({
+      assetsMap: new Map(),
+      paths: [],
+      archiveIterator: {},
+    } as any);
+    MockArchive.generatePackageInfoFromArchiveBuffer.mockReturnValue({
+      packageInfo: { name: 'testpkg', version: '1.0.1' },
+    } as any);
+
+    mockGetBundledPackageByName.mockResolvedValue(bundledPackage);
+    await expect(getPackage('testpkg', '1.0.1')).rejects.toThrowError(
+      new FleetError('Error fetching package')
+    );
+  });
+
+  it('should try to retrieve from cache if there is a RegistryConnectionError and no bundled package', async () => {
+    mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage]));
+    mockGetResponseStreamWithSize.mockRejectedValueOnce(
+      new RegistryConnectionError('Error connecting to EPR')
+    );
+    mockStreamToBuffer.mockResolvedValue(Buffer.from('testpkg'));
+    mockVerifyPackageArchiveSignature.mockResolvedValue('verified');
+    MockArchive.unpackBufferToAssetsMap.mockReturnValue({
+      assetsMap: new Map([
+        ['test-1.0.0/LICENSE.txt', Buffer.from('')],
+        ['test-1.0.0/changelog.yml', Buffer.from('')],
+        ['test-1.0.0/manifest.yml', Buffer.from('')],
+        ['test-1.0.0/docs/README.md', Buffer.from('')],
+      ]),
+      paths: [],
+      archiveIterator: {},
+    } as any);
+    MockArchive.getPackageInfo.mockReturnValue({ name: 'testpkg', version: '1.0.1' } as any);
+    mockGetPackageAssetsMapCache.mockReturnValue({
+      name: 'test',
+    } as any);
+    mockGetBundledPackageByName.mockResolvedValue(bundledPackage);
+    const result = await getPackage('testpkg', '1.0.1');
+    expect(result).toEqual({
+      archiveIterator: expect.any(Object),
+      assetsMap: new Map([
+        ['test-1.0.0/LICENSE.txt', Buffer.from('')],
+        ['test-1.0.0/changelog.yml', Buffer.from('')],
+        ['test-1.0.0/manifest.yml', Buffer.from('')],
+        ['test-1.0.0/docs/README.md', Buffer.from('')],
+      ]),
+      packageInfo: {
+        name: 'testpkg',
+        version: '1.0.1',
+      },
+      paths: [],
+      verificationResult: undefined,
+    });
+  });
+
+  it('should falls back to bundled package if there is a RegistryConnectionError', async () => {
+    mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage]));
+    mockGetResponseStreamWithSize.mockRejectedValueOnce(
+      new RegistryConnectionError('Error connecting to EPR')
+    );
+    mockStreamToBuffer.mockResolvedValue(Buffer.from('testpkg'));
+    mockVerifyPackageArchiveSignature.mockResolvedValue('verified');
+    MockArchive.unpackBufferToAssetsMap.mockReturnValue({
+      assetsMap: new Map([
+        ['test-1.0.0/LICENSE.txt', Buffer.from('')],
+        ['test-1.0.0/changelog.yml', Buffer.from('')],
+        ['test-1.0.0/manifest.yml', Buffer.from('')],
+        ['test-1.0.0/docs/README.md', Buffer.from('')],
+      ]),
+      paths: [],
+      archiveIterator: {},
+    } as any);
+    MockArchive.getPackageInfo.mockReturnValue({ name: 'testpkg', version: '1.0.1' } as any);
+    mockGetPackageAssetsMapCache.mockReturnValue(new Map());
+    mockGetBundledPackageByName.mockResolvedValue(bundledPackage);
+    const result = await getPackage('testpkg', '1.0.1');
+    expect(result).toEqual({
+      archiveIterator: expect.any(Object),
+      assetsMap: new Map([
+        ['test-1.0.0/LICENSE.txt', Buffer.from('')],
+        ['test-1.0.0/changelog.yml', Buffer.from('')],
+        ['test-1.0.0/manifest.yml', Buffer.from('')],
+        ['test-1.0.0/docs/README.md', Buffer.from('')],
+      ]),
+      packageInfo: {
+        name: 'testpkg',
+        version: '1.0.1',
+      },
+      paths: [],
+      verificationResult: undefined,
+    });
+  });
+
+  it('should throw if there is a RegistryConnectionError and could not find bundled package nor retrieve from cache ', async () => {
+    mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage]));
+    mockGetResponseStreamWithSize.mockRejectedValueOnce(
+      new RegistryConnectionError('Error connecting to EPR')
+    );
+    mockStreamToBuffer.mockResolvedValue(Buffer.from('testpkg'));
+    mockVerifyPackageArchiveSignature.mockResolvedValue('verified');
+    MockArchive.unpackBufferToAssetsMap.mockReturnValue({
+      assetsMap: new Map(),
+      paths: [],
+      archiveIterator: {},
+    } as any);
+    MockArchive.generatePackageInfoFromArchiveBuffer.mockReturnValue({
+      packageInfo: undefined,
+    } as any);
+
+    mockGetBundledPackageByName.mockResolvedValue(undefined);
+    await expect(getPackage('testpkg', '1.0.1')).rejects.toThrowError(
+      new PackageNotFoundError('[email protected] not found')
+    );
+  });
+});