[Fleet] Add retry logic to automatic agent upgrades (elastic#212744)

Relates elastic/ingest-dev#4720

This PR adds retry logic to the task that handles automatic agent
upgrades originally implemented in
elastic#211019.

Complementary fleet-server change which sets the agent's
`upgrade_attempts` to `null` once the upgrade is complete.:
elastic/fleet-server#4528

- A new `upgrade_attempts` property is added to agents and stored in the
agent doc (ES mapping update in
elastic/elasticsearch#123256).
- When a bulk upgrade action is sent from the automatic upgrade task, it
pushes the timestamp of the upgrade to the affected agents'
`upgrade_attempts`.
- The default retry delays are `['30m', '1h', '2h', '4h', '8h', '16h',
'24h']` and can be overridden with the new
`xpack.fleet.autoUpgrades.retryDelays` setting.
- On every run, the automatic upgrade task will first process retries
and then query more agents if necessary (cf.
elastic/ingest-dev#4720 (comment)).
- Once an agent has completed and failed the max retries defined by the
retry delays array, it is no longer retried.

The ES query for fetching agents with existing `upgrade_attempts` needs
the updated mappings, so it might be necessary to pull the latest `main`
in the `elasticsearch` repo and run `yarn es source` instead of `yarn es
snapshot` (requires an up-to-date Java environment, currently 23).

In order to test that `upgrade_attempts` is set to `null` when the
upgrade is complete, fleet-server should be run in dev using the change
in elastic/fleet-server#4528.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

Low probability risk of incorrectly triggering agent upgrades. This
feature is currently behind the `enableAutomaticAgentUpgrades` feature
flag.

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: Julia Bardi <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>

Author: 4 people

oas_docs/output/kibana.serverless.yaml

@@ -19330,6 +19330,11 @@ paths:
                             type: string
                           nullable: true
                           type: array
+                        upgrade_attempts:
+                          items:
+                            type: string
+                          nullable: true
+                          type: array
                         upgrade_details:
                           additionalProperties: false
                           type: object
@@ -19794,6 +19799,11 @@ paths:
                           type: string
                         nullable: true
                         type: array
+                      upgrade_attempts:
+                        items:
+                          type: string
+                        nullable: true
+                        type: array
                       upgrade_details:
                         additionalProperties: false
                         type: object
@@ -20137,6 +20147,11 @@ paths:
                           type: string
                         nullable: true
                         type: array
+                      upgrade_attempts:
+                        items:
+                          type: string
+                        nullable: true
+                        type: array
                       upgrade_details:
                         additionalProperties: false
                         type: object

x-pack/platform/plugins/shared/fleet/common/constants/index.ts

@@ -58,3 +58,5 @@ export const FLEET_ENROLLMENT_API_PREFIX = 'fleet-enrollment-api-keys';
 export const REQUEST_DIAGNOSTICS_TIMEOUT_MS = 3 * 60 * 60 * 1000; // 3 hours;
 
 export * from './mappings';
+
+export const AUTO_UPGRADE_DEFAULT_RETRIES = ['30m', '1h', '2h', '4h', '8h', '16h', '24h'];

x-pack/platform/plugins/shared/fleet/common/constants/mappings.ts

@@ -363,6 +363,9 @@ export const AGENT_MAPPINGS = {
         },
       },
     },
+    upgrade_attempts: {
+      type: 'date',
+    },
     // added to allow validation on status field
     status: {
       type: 'keyword',

x-pack/platform/plugins/shared/fleet/common/types/index.ts

@@ -83,6 +83,9 @@ export interface FleetConfigType {
     };
   };
   createArtifactsBulkBatchSize?: number;
+  autoUpgrades?: {
+    retryDelays?: string[];
+  };
 }
 
 // Calling Object.entries(PackagesGroupedByStatus) gave `status: string`

x-pack/platform/plugins/shared/fleet/common/types/models/agent.ts

@@ -96,6 +96,7 @@ interface AgentBase {
   upgraded_at?: string | null;
   upgrade_started_at?: string | null;
   upgrade_details?: AgentUpgradeDetails;
+  upgrade_attempts?: string[] | null;
   access_api_key_id?: string;
   default_api_key?: string;
   default_api_key_id?: string;
@@ -268,6 +269,10 @@ export interface FleetServerAgent {
    * Upgrade state of the Elastic Agent
    */
   upgrade_details?: AgentUpgradeDetails;
+  /**
+   * List of timestamps of attempts of Elastic Agent automatic upgrades
+   */
+  upgrade_attempts?: string[] | null;
   access_api_key_id?: string;
   agent?: FleetServerAgentMetadata;
   /**

x-pack/platform/plugins/shared/fleet/public/applications/fleet/components/search_bar.test.tsx

@@ -174,7 +174,7 @@ describe('SearchBar', () => {
 
 describe('getFieldSpecs', () => {
   it('returns fieldSpecs for Fleet agents', () => {
-    expect(getFieldSpecs(AGENTS_INDEX, AGENTS_PREFIX)).toHaveLength(73);
+    expect(getFieldSpecs(AGENTS_INDEX, AGENTS_PREFIX)).toHaveLength(74);
   });
 
   it('returns fieldSpecs for Fleet enrollment tokens', () => {

x-pack/platform/plugins/shared/fleet/server/config.ts

@@ -283,6 +283,11 @@ export const config: PluginConfigDescriptor = {
           min: 400,
         })
       ),
+      autoUpgrades: schema.maybe(
+        schema.object({
+          retryDelays: schema.maybe(schema.arrayOf(schema.string())),
+        })
+      ),
     },
     {
       validate: (configToValidate) => {

x-pack/platform/plugins/shared/fleet/server/services/agents/helpers.ts

@@ -68,6 +68,7 @@ export function searchHitToAgent(
     upgraded_at: hit._source?.upgraded_at,
     upgrade_started_at: hit._source?.upgrade_started_at,
     upgrade_details: hit._source?.upgrade_details,
+    upgrade_attempts: hit._source?.upgrade_attempts,
     access_api_key_id: hit._source?.access_api_key_id,
     default_api_key_id: hit._source?.default_api_key_id,
     policy_id: hit._source?.policy_id,

x-pack/platform/plugins/shared/fleet/server/services/agents/upgrade.ts

@@ -125,3 +125,22 @@ export async function sendUpgradeAgentsActions(
 
   return await upgradeBatch(esClient, givenAgents, outgoingErrors, options, currentSpaceId);
 }
+
+export async function sendAutomaticUpgradeAgentsActions(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  options: {
+    agents: Agent[];
+    version: string;
+    upgradeDurationSeconds?: number;
+  }
+): Promise<{ actionId: string }> {
+  const currentSpaceId = getCurrentNamespace(soClient);
+  return await upgradeBatch(
+    esClient,
+    options.agents,
+    {},
+    { ...options, isAutomatic: true },
+    currentSpaceId
+  );
+}

x-pack/platform/plugins/shared/fleet/server/services/agents/upgrade_action_runner.ts

@@ -9,11 +9,13 @@ import type { ElasticsearchClient } from '@kbn/core/server';
 
 import { v4 as uuidv4 } from 'uuid';
 import moment from 'moment';
+import semverGte from 'semver/functions/gte';
 
 import {
   getRecentUpgradeInfoForAgent,
   getNotUpgradeableMessage,
   isAgentUpgradeableToVersion,
+  AGENT_UPGARDE_DETAILS_SUPPORTED_VERSION,
 } from '../../../common/services';
 
 import type { Agent } from '../../types';
@@ -168,6 +170,10 @@ export async function upgradeBatch(
       data: {
         upgraded_at: null,
         upgrade_started_at: now,
+        ...(options.isAutomatic &&
+        semverGte(agent.agent?.version ?? '0.0.0', AGENT_UPGARDE_DETAILS_SUPPORTED_VERSION)
+          ? { upgrade_attempts: [now, ...(agent.upgrade_attempts ?? [])] }
+          : {}),
       },
     })),
     errors