Add a configurable parameter to the number of backup attempts that are made.

agent/pom.xml

@@ -4,12 +4,12 @@
     <parent>
         <groupId>com.orientechnologies</groupId>
         <artifactId>orientdb-enterprise-parent</artifactId>
-        <version>3.2.50-SNAPSHOT</version>
+        <version>3.2.49</version>
         <relativePath>../</relativePath>
     </parent>
     <groupId>com.orientechnologies</groupId>
     <artifactId>agent</artifactId>
-    <version>3.2.50-SNAPSHOT</version>
+    <version>3.2.49</version>
     <packaging>jar</packaging>
     <name>OrientDB Agent</name>
     <description> OrientDB Enterprise Agent </description>

agent/src/main/java/com/orientechnologies/agent/services/backup/OBackupConfig.java

@@ -50,6 +50,7 @@ public class OBackupConfig {
   public static final String DIRECTORY = "directory";
   public static final String MODES = "modes";
   public static final String ID = "uuid";
+  public static final String RETRIES = "retries";
 
   private static final String configFile = "${ORIENTDB_HOME}/config/backups.json";
   private String filePath = null;

agent/src/main/java/com/orientechnologies/agent/services/backup/OBackupTask.java

@@ -35,6 +35,7 @@ public class OBackupTask implements OBackupListener {
   private TimerTask task;
   private OBackupListener listener;
   private OEnterpriseServer server;
+  private int currentRetryCount = 0;
 
   public OBackupTask(OBackupStrategy strategy, OEnterpriseServer server) {
     this.strategy = strategy;
@@ -93,6 +94,27 @@ public OBackupStrategy getStrategy() {
     return strategy;
   }
 
+  public int getCurrentRetryCount() {
+    return currentRetryCount;
+  }
+
+  private void resetRetryCount() {
+    this.currentRetryCount = 0;
+  }
+
+  private long calculateBackoffDelay() {
+    switch (currentRetryCount) {
+      case 0:
+        return 60 * 1000L; // 1 minute
+      case 1:
+        return 5 * 60 * 1000L; // 5 minutes
+      case 2:
+        return 15 * 60 * 1000L; // 15 minutes
+      default:
+        return 30 * 60 * 1000L; // 30 minutes
+    }
+  }
+
   public void changeConfig(final OBackupConfig config, final ODocument doc) {
     if (task != null) {
       task.cancel();
@@ -110,12 +132,64 @@ public void changeConfig(final OBackupConfig config, final ODocument doc) {
   @Override
   public Boolean onEvent(final ODocument cfg, final OBackupLog log) {
     final boolean canContinue = invokeListener(cfg, log);
-    if (OBackupLogType.BACKUP_FINISHED.equals(log.getType())
-        || OBackupLogType.BACKUP_ERROR.equals(log.getType())) {
+
+    if (OBackupLogType.BACKUP_FINISHED.equals(log.getType())) {
+      resetRetryCount();
       if (canContinue) {
         schedule();
       }
+    } else if (OBackupLogType.BACKUP_ERROR.equals(log.getType())) {
+      final int maxRetries = strategy.getRetriesWithDefault();
+
+      if (currentRetryCount < maxRetries) {
+        final long delay = calculateBackoffDelay();
+        currentRetryCount++;
+
+        OLogManager.instance()
+            .warn(
+                this,
+                "Backup failed for ["
+                    + strategy.getDbName()
+                    + "]. Retry attempt "
+                    + currentRetryCount
+                    + "/"
+                    + maxRetries
+                    + " scheduled in "
+                    + (delay / 1000)
+                    + " seconds");
+
+        task =
+            Orient.instance()
+                .scheduleTask(
+                    () -> {
+                      server
+                          .getDatabases()
+                          .execute(
+                              () -> {
+                                try {
+                                  final long start = tickStart();
+                                  strategy.doBackup(OBackupTask.this);
+                                  tickEnd(start);
+                                } catch (final IOException e) {
+                                  OLogManager.instance().error(this, "Error " + e.getMessage(), e);
+                                }
+                              });
+                    },
+                    new Date(System.currentTimeMillis() + delay),
+                    0);
+      } else {
+        OLogManager.instance()
+            .warn(
+                this,
+                "Backup failed for ["
+                    + strategy.getDbName()
+                    + "] after "
+                    + maxRetries
+                    + " retries. Manual intervention required. Backup scheduling stopped.");
+        currentRetryCount = 0;
+      }
     }
+
     return true;
   }
 

agent/src/main/java/com/orientechnologies/agent/services/backup/log/OBackupErrorLog.java

@@ -25,6 +25,7 @@ public class OBackupErrorLog extends OBackupLog {
 
   protected String message;
   protected String stackTrace;
+  protected Integer retryCount;
 
   public OBackupErrorLog(long unitId, long opsId, String uuid, String dbName, String mode) {
     super(unitId, opsId, uuid, dbName, mode);
@@ -35,6 +36,7 @@ public ODocument toDoc() {
     ODocument document = super.toDoc();
     document.field("message", message);
     document.field("stackTrace", stackTrace);
+    document.field("retryCount", retryCount);
     return document;
   }
 
@@ -43,6 +45,7 @@ public void fromDoc(ODocument doc) {
     super.fromDoc(doc);
     this.message = doc.field("message");
     this.stackTrace = doc.field("stackTrace");
+    this.retryCount = doc.field("retryCount");
   }
 
   @Override
@@ -65,4 +68,12 @@ public String getMessage() {
   public String getStackTrace() {
     return stackTrace;
   }
+
+  public Integer getRetryCount() {
+    return retryCount;
+  }
+
+  public void setRetryCount(Integer retryCount) {
+    this.retryCount = retryCount;
+  }
 }

agent/src/main/java/com/orientechnologies/agent/services/backup/strategy/OBackupStrategy.java

@@ -88,6 +88,12 @@ public void doBackup(final OBackupListener listener) throws IOException {
       final StringWriter sw = new StringWriter();
       error.setMessage(e.getMessage());
       error.setStackTrace(sw.toString());
+
+      // Add retry count if listener is OBackupTask
+      if (listener instanceof OBackupTask) {
+        error.setRetryCount(((OBackupTask) listener).getCurrentRetryCount());
+      }
+
       logger.log(error);
       listener.onEvent(cfg, error);
       return;
@@ -321,6 +327,15 @@ public Integer getRetentionDays() {
     return cfg.field(OBackupConfig.RETENTION_DAYS);
   }
 
+  public Integer getRetries() {
+    return cfg.field(OBackupConfig.RETRIES);
+  }
+
+  public int getRetriesWithDefault() {
+    Integer retries = getRetries();
+    return (retries != null && retries >= 0) ? retries : 3;
+  }
+
   protected OBackupScheduledLog lastUnfiredSchedule() {
     OBackupLog lastSchedule = null;
     try {

agent/src/test/java/com/orientechnologies/agent/backup/OBackupServiceBigTest.java

@@ -104,6 +104,9 @@ public void bootOrientDB() throws Exception {
     server.startup(stream);
 
     orient = server.getContext();
+    if (orient.exists(DB_NAME)) {
+      orient.drop(DB_NAME);
+    }
     orient.execute(
         "create database " + DB_NAME + " plocal users(admin identified by 'admin' role admin)");
     server.activate();

pom.xml

@@ -3,11 +3,11 @@
     <parent>
         <groupId>com.orientechnologies</groupId>
         <artifactId>orientdb-parent</artifactId>
-        <version>3.2.50-SNAPSHOT</version>
+        <version>3.2.49</version>
     </parent>
     <groupId>com.orientechnologies</groupId>
     <artifactId>orientdb-enterprise-parent</artifactId>
-    <version>3.2.50-SNAPSHOT</version>
+    <version>3.2.49</version>
     <packaging>pom</packaging>
     <name>OrientDB Agent Parent</name>
     <description> OrientDB Enterprise Parent </description>