feat(gorgone-autodisco): implement a retry for autodiscovery host cron job (#3121) (#3122)

* feat(gorgone-autodisco): Add retry mechanism for automatic host discovery

Manual discovery are not changed.
automatic discovery are passed to status failed in this function, and the following code will run the discovery just as normal

* ci(gorgone-autodisco): Add new dep to parse date

This is a pretty common lib, useful for ingesting date from other sources

Co-authored-by: Sylvain Cresto <scresto@centreon.com>

Refs:MON-176449

Co-authored-by: pkippes <144150042+pkippes@users.noreply.github.com>

Author: Evan-Adam

gorgone/gorgone/modules/centreon/autodiscovery/class.pm

@@ -35,6 +35,8 @@ use gorgone::class::frame;
 use JSON::XS;
 use Time::HiRes;
 use POSIX qw(strftime);
+use DateTime;
+use DateTime::Format::Strptime;
 use Digest::MD5 qw(md5_hex);
 use Try::Tiny;
 use EV;
@@ -149,6 +151,125 @@ sub hdisco_is_running_job {
 
     return 0;
 }
+=head3 $self->hdisco_can_start_job(job => $jobRef)
+
+Check if we can start a host discovery job.
+If the job is in timeout, update the job status in db and run the job again
+
+For now there is no real mutex on the execution except the db column status,
+so if the post execution command outlive the timeout undefined behaviour may appear.
+
+Parameters:
+
+=over 4
+
+=item * job: job information hash ref. Required data to work correctly are :
+
+=over 4
+
+=item * status: the job status (see JOB_FINISH and other constant for possible states)
+
+=item * job_id: the job unique identifier. Used to set the job as failure if timeout is reached.
+
+=item * last_execution : hash containing 'timezone' and 'date' (ex 'Europe/Paris',  '2026-01-30 20:31:00.000000')
+
+=back
+
+=back
+
+Output : Bool
+
+1 if the job should be started
+
+0 if invalid data given, or job correctly running and should not be started.
+=cut
+sub hdisco_can_start_job {
+    my ($self, %options) = @_;
+    if (!$options{job} || !defined($options{job}->{status}) || !defined($options{job}->{job_id})){
+        return 0;
+    }
+    if ($options{job}->{status} != JOB_RUNNING &&
+        $options{job}->{status} != SAVE_RUNNING) {
+        # if job is not running, we can start it safely
+        return 1;
+    }
+    if (!defined($options{job}->{last_execution}) || !defined($options{job}->{last_execution}->{date})) {
+        # probably first run of the job
+        return 1;
+    }
+    if ($options{job}->{execution}->{mode} != 1){
+        # never timeout manual or paused jobs, only retry automatic jobs
+        return 0;
+    }
+
+    my $second_since_last_exec = $self->_get_duration_since_last_exec(
+        date     => $options{job}->{last_execution}->{date},
+        timezone => $options{job}->{last_execution}->{timezone} // 'UTC'
+    ) or return 0; # could not parse the date, don't try to start the job again.
+
+    my $timeout = $options{timeout} // $self->{global_timeout};
+    if ($second_since_last_exec <= $timeout * 2 + 10) {
+        # job did not reach timeout, let it run, don't start it again.
+        return 0;
+    }
+
+    # job is in timeout, restarting it.
+    $self->{logger}->writeLogError(
+        "[autodiscovery] job is timing out (last execution: '" . $options{job}->{last_execution}->{date} . "'), we set it as failed and restart it");
+    return 0 if -1 == $self->update_job_information(
+        values       => {
+            status  => JOB_FAILED,
+            message => 'Job timed out and will be restarted by Gorgone',
+        },
+        where_clause => [
+            { id => $options{job}->{job_id} }
+        ]);
+    $self->{hdisco_jobs_ids}->{ $options{job}->{job_id} }->{status} = JOB_FAILED;
+    return 1;
+
+}
+=head3 $self->_get_duration_since_last_exec(date => $dateStr, timezone => $tzStr)
+
+Calculate the duration in seconds since the last job execution.
+
+Parameters:
+
+=over 4
+
+=item * date: execution date string in format 'YYYY-MM-DD HH:MM:SS.NNNNNN' (e.g., '2026-01-30 20:31:00.000000')
+
+=item * timezone: timezone string for the date (e.g., 'Europe/Paris', 'UTC')
+
+=back
+
+Output : Int or undef
+
+Number of seconds since the last execution.
+
+Returns undef if the date cannot be parsed or if the date is in the future.
+
+=cut
+sub _get_duration_since_last_exec {
+    my ($self, %options) = @_;
+
+    # parse format "2026-01-29 15:35:12.000000" given by php api.
+    my $last_exec = DateTime::Format::Strptime->new(
+        pattern   => '%Y-%m-%d %H:%M:%S.%6N',
+        time_zone => $options{timezone},
+    )->parse_datetime($options{date}); # will return undef on any failure.
+
+    if (!defined($last_exec)) {
+        $self->{logger}->writeLogWarning("[autodiscovery] can not parse last execution date '" . $options{date} . "from job, job won't start.");
+        return undef;
+    }
+
+    my $duration = DateTime->now()->epoch() - $last_exec->epoch();
+    if ($duration < 0) {
+        $self->{logger}->writeLogWarning("[autodiscovery] last execution date '" . $options{date} . "' is in the future, job won't start.");
+        return undef;
+    }
+    return $duration;
+}
 
 sub hdisco_add_cron {
     my ($self, %options) = @_;
@@ -443,7 +564,7 @@ sub launchhostdiscovery {
     if (!defined($job_id) || !defined($self->{hdisco_jobs_ids}->{$job_id})) {
         return (1, 'trying to launch discovery for inexistant job');
     }
-    if ($self->hdisco_is_running_job(status => $self->{hdisco_jobs_ids}->{$job_id}->{status})) {
+    if (! $self->hdisco_can_start_job(job => $self->{hdisco_jobs_ids}->{$job_id})) {
         return (1, 'job is already running');
     }
     if ($self->{hdisco_jobs_ids}->{$job_id}->{execution}->{mode} == EXECUTION_MODE_PAUSE && $options{source} eq 'cron') {
@@ -927,7 +1048,7 @@ sub update_job_information {
 
     my ($status) = $self->{class_object_centreon}->custom_execute(request => $query, bind_values => \@bind_values);
     if ($status == -1) {
-        $self->{logger}->writeLogError('[autodiscovery] Failed to update job information');
+        $self->{logger}->writeLogError('[autodiscovery] Failed to update job information for ' . join(', ', @bind_values));
         return -1;
     }
 

gorgone/packaging/centreon-gorgone.yaml

@@ -198,6 +198,7 @@ overrides:
       - perl(lib)
       - perl(Safe)
       - perl(Tie::File) # required by MBI module
+      - perl(DateTime::Format::Strptime)
     recommends:
       - logrotate
 
@@ -234,6 +235,7 @@ overrides:
       - libzmq-ffi-perl
       - libclone-choose-perl
       - libjson-perl # gorgone_key_thumbprint.pl needs the json module, even when json::xs is already installed
+      - libdatetime-format-strptime-perl # autodiscovery uses it to retry job scheduling
       - librrds-perl
       - perl-base
       - perl-modules

gorgone/tests/unit/modules/centreon/autodiscovery/host.t

@@ -0,0 +1,73 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use Test2::V0;
+use Test2::Plugin::NoWarnings echo => 1;
+use Test2::Tools::Compare qw{is like match};
+use FindBin;
+use lib "$FindBin::Bin/../../../../../";
+use lib "$FindBin::Bin/../../../../../../perl-libs/lib/";
+use Data::Dumper;
+use gorgone::modules::centreon::autodiscovery::class;
+use gorgone::modules::core::action::class;
+use tests::unit::lib::mockLogger;
+
+sub main {
+    test_hdisco_can_start_job();
+    done_testing();
+}
+sub test_hdisco_can_start_job {
+    my $mock_logger = mock 'centreon::common::logger';
+
+    my $gorgone = bless
+        { logger => $mock_logger->class,
+          global_timeout => 60 },
+        "gorgone::modules::centreon::autodiscovery::class";
+
+    my @tests = ({
+        name => "first",
+        arg    => {
+            status   => 4,
+            timezone => 'Europe/Paris',
+            date     => '2026-01-29 15:35:12.000000',
+            timeout  => 60 },
+        result => 1,
+
+    }, {
+        name   => "undef mean take default timeout",
+        arg    => {
+            status   => 4,
+            timezone => 'Europe/Paris',
+            date     => '2026-01-29 15:35:12.000000',
+            timeout  => undef },
+        result => 1
+    },
+    );
+    for my $test (@tests) {
+        my $mock_gorgone = mock('gorgone::modules::centreon::autodiscovery::class' => (override => [
+            _get_duration_since_last_exec => sub {
+                return $test->{args}->{duration} // 300; # limit is at 130 secs
+            }, update_job_information => sub {
+                return 1;
+        }]));
+        my $res = $gorgone->hdisco_can_start_job(
+            "timeout" => $test->{arg}->{timeout},
+            "job"     => {
+                job_id => 2,
+                execution => {mode => 1},
+                status         => $test->{arg}->{status},
+                last_execution => {
+                    "timezone_type" => 3,
+                    'timezone'      => $test->{arg}->{timezone},
+                    'date'          => $test->{arg}->{date},
+                    'timezone_type' => 3
+                }
+            });
+        is($res, $test->{result}, $test->{name});
+    }
+
+}
+
+main();
+