[Fleet] Add retry logic to automatic agent upgrades

docs/settings/fleet-settings.asciidoc

@@ -57,6 +57,9 @@ Hostnames used by {agent} for accessing {es}.
 `xpack.fleet.agents.elasticsearch.ca_sha256`::
 Hash pin used for certificate verification. The pin is a base64-encoded string of the SHA-256 fingerprint.
 
+`xpack.fleet.autoUpgrades.retryDelays`::
+List of delays for retrying failed automatic agent upgrades. Overrides default exponential backoff strategy.
+
 [role="child_attributes"]
 ==== Preconfiguration settings (for advanced use cases)
 

oas_docs/output/kibana.serverless.yaml

@@ -18928,6 +18928,11 @@ paths:
                         upgraded_at:
                           nullable: true
                           type: string
+                        upgrade_attempts:
+                          nullable: true
+                          type: array
+                          items:
+                            type: string
                         user_provided_metadata:
                           additionalProperties: {}
                           type: object
@@ -19383,6 +19388,11 @@ paths:
                       upgraded_at:
                         nullable: true
                         type: string
+                      upgrade_attempts:
+                          nullable: true
+                          type: array
+                          items:
+                            type: string
                       user_provided_metadata:
                         additionalProperties: {}
                         type: object
@@ -19726,6 +19736,11 @@ paths:
                       upgraded_at:
                         nullable: true
                         type: string
+                      upgrade_attempts:
+                          nullable: true
+                          type: array
+                          items:
+                            type: string
                       user_provided_metadata:
                         additionalProperties: {}
                         type: object

oas_docs/output/kibana.yaml

@@ -21049,6 +21049,11 @@ paths:
                         upgraded_at:
                           nullable: true
                           type: string
+                        upgrade_attempts:
+                          nullable: true
+                          type: array
+                          items:
+                            type: string
                         user_provided_metadata:
                           additionalProperties: {}
                           type: object
@@ -21501,6 +21506,11 @@ paths:
                       upgraded_at:
                         nullable: true
                         type: string
+                      upgrade_attempts:
+                          nullable: true
+                          type: array
+                          items:
+                            type: string
                       user_provided_metadata:
                         additionalProperties: {}
                         type: object
@@ -21843,6 +21853,11 @@ paths:
                       upgraded_at:
                         nullable: true
                         type: string
+                      upgrade_attempts:
+                          nullable: true
+                          type: array
+                          items:
+                            type: string
                       user_provided_metadata:
                         additionalProperties: {}
                         type: object

x-pack/platform/plugins/shared/fleet/common/constants/index.ts

@@ -58,3 +58,5 @@ export const FLEET_ENROLLMENT_API_PREFIX = 'fleet-enrollment-api-keys';
 export const REQUEST_DIAGNOSTICS_TIMEOUT_MS = 3 * 60 * 60 * 1000; // 3 hours;
 
 export * from './mappings';
+
+export const AUTO_UPGRADE_DEFAULT_RETRIES = ['30m', '1h', '2h', '4h', '8h', '16h', '24h'];

x-pack/platform/plugins/shared/fleet/common/constants/mappings.ts

@@ -363,6 +363,9 @@ export const AGENT_MAPPINGS = {
         },
       },
     },
+    upgrade_attempts: {
+      type: 'date',
+    },
     // added to allow validation on status field
     status: {
       type: 'keyword',

x-pack/platform/plugins/shared/fleet/common/types/index.ts

@@ -84,6 +84,9 @@ export interface FleetConfigType {
     };
   };
   createArtifactsBulkBatchSize?: number;
+  autoUpgrades: {
+    retryDelays: string[];
+  };
 }
 
 // Calling Object.entries(PackagesGroupedByStatus) gave `status: string`

x-pack/platform/plugins/shared/fleet/common/types/models/agent.ts

@@ -99,6 +99,7 @@ interface AgentBase {
   upgraded_at?: string | null;
   upgrade_started_at?: string | null;
   upgrade_details?: AgentUpgradeDetails;
+  upgrade_attempts?: string[] | null;
   access_api_key_id?: string;
   default_api_key?: string;
   default_api_key_id?: string;
@@ -275,6 +276,10 @@ export interface FleetServerAgent {
    * Upgrade state of the Elastic Agent
    */
   upgrade_details?: AgentUpgradeDetails;
+  /**
+   * List of timestamps of attempts of Elastic Agent automatic upgrades
+   */
+  upgrade_attempts?: string[] | null;
   access_api_key_id?: string;
   agent?: FleetServerAgentMetadata;
   /**

x-pack/platform/plugins/shared/fleet/server/config.ts

@@ -12,6 +12,7 @@ import type { TypeOf } from '@kbn/config-schema';
 import type { PluginConfigDescriptor } from '@kbn/core/server';
 
 import { isValidExperimentalValue } from '../common/experimental_features';
+import { AUTO_UPGRADE_DEFAULT_RETRIES } from '../common/constants';
 
 import {
   PreconfiguredPackagesSchema,
@@ -282,6 +283,11 @@ export const config: PluginConfigDescriptor = {
           min: 400,
         })
       ),
+      autoUpgrades: schema.object({
+        retryDelays: schema.arrayOf(schema.string(), {
+          defaultValue: AUTO_UPGRADE_DEFAULT_RETRIES,
+        }),
+      }),
     },
     {
       validate: (configToValidate) => {

x-pack/platform/plugins/shared/fleet/server/services/agents/helpers.ts

@@ -69,6 +69,7 @@ export function searchHitToAgent(
     upgraded_at: hit._source?.upgraded_at,
     upgrade_started_at: hit._source?.upgrade_started_at,
     upgrade_details: hit._source?.upgrade_details,
+    upgrade_attempts: hit._source?.upgrade_attempts,
     access_api_key_id: hit._source?.access_api_key_id,
     default_api_key_id: hit._source?.default_api_key_id,
     policy_id: hit._source?.policy_id,

x-pack/platform/plugins/shared/fleet/server/services/agents/upgrade.ts

@@ -125,3 +125,22 @@ export async function sendUpgradeAgentsActions(
 
   return await upgradeBatch(esClient, givenAgents, outgoingErrors, options, currentSpaceId);
 }
+
+export async function sendAutomaticUpgradeAgentsActions(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  options: {
+    agents: Agent[];
+    version: string;
+    upgradeDurationSeconds?: number;
+  }
+): Promise<{ actionId: string }> {
+  const currentSpaceId = getCurrentNamespace(soClient);
+  return await upgradeBatch(
+    esClient,
+    options.agents,
+    {},
+    { ...options, isAutomatic: true },
+    currentSpaceId
+  );
+}

x-pack/platform/plugins/shared/fleet/server/services/agents/upgrade_action_runner.ts

@@ -168,6 +168,9 @@ export async function upgradeBatch(
       data: {
         upgraded_at: null,
         upgrade_started_at: now,
+        ...(options.isAutomatic
+          ? { upgrade_attempts: [...(agent.upgrade_attempts ?? []), now] }
+          : {}),
       },
     })),
     errors