#!/usr/bin/perl

use strict;
use warnings;
use DBI();
use OAR::IO;
use OAR::Modules::Judas qw(oar_debug oar_warn oar_info oar_error set_current_log_category);
use OAR::Conf qw(init_conf dump_conf get_conf is_conf get_conf_with_default_param);
use Data::Dumper;
use OAR::Tools;
use OAR::Modules::Hulot;
use OAR::Schedulers::GanttHoleStorage;
use OAR::Walltime;

my $Module_name = "MetaSched";
my $Session_id  = $$;

# Log category
set_current_log_category('scheduler');

my $exit_code = 0;
my $base      = OAR::IO::connect();
my $base_ro   = OAR::IO::connect_ro();

init_conf($ENV{OARCONFFILE});
my $Order_part = get_conf_with_default_param("SCHEDULER_RESOURCE_ORDER", "");
my $Order_part_ar =
  get_conf_with_default_param("SCHEDULER_RESOURCE_ORDER_ADV_RESERVATIONS", $Order_part);
my $Order_part_ar_threshold =
  get_conf_with_default_param("SCHEDULER_RESOURCE_ORDER_ADV_RESERVATIONS_THRESHOLD", 300);

my $Walltime_change_cancellation_timeout = OAR::Walltime::get_default_timeout();

my $binpath;
if (defined($ENV{OARDIR})) {
    $binpath = $ENV{OARDIR} . "/";
} else {
    die("[MetaSched] Error: OARDIR env variable must be defined\n");
}

my $Cpuset_field = get_conf("JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD");
my $Remote_host  = get_conf("SERVER_HOSTNAME");
my $Remote_port  = get_conf("SERVER_PORT");

my @Sched_available_suspended_resource_type;
my $sched_available_suspended_resource_type_tmp =
  get_conf("SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE");
if (!defined($sched_available_suspended_resource_type_tmp)) {
    push(@Sched_available_suspended_resource_type, "default");
} else {
    @Sched_available_suspended_resource_type =
      split(" ", $sched_available_suspended_resource_type_tmp);
}

# Look at resources that we must add for each job
my $Resources_to_always_add_type = get_conf("SCHEDULER_RESOURCES_ALWAYS_ASSIGNED_TYPE");
my @Resources_to_always_add      = ();

# Do we log every scheduler's computation ?
my $Log_scheduling = get_conf("SCHEDULER_LOG_DECISIONS");
if (defined($Log_scheduling) and $Log_scheduling ne "yes") { $Log_scheduling = undef; }

#minimum of seconds between each jobs
my $Security_time_overhead = get_conf_with_default_param("SCHEDULER_JOB_SECURITY_TIME", 1);

my $Openssh_cmd = get_conf("OPENSSH_CMD");
$Openssh_cmd = OAR::Tools::get_default_openssh_cmd() if (!defined($Openssh_cmd));
if (is_conf("OAR_SSH_CONNECTION_TIMEOUT")) {
    OAR::Tools::set_ssh_timeout(get_conf("OAR_SSH_CONNECTION_TIMEOUT"));
}
my $Server_hostname = get_conf("SERVER_HOSTNAME");
my $Deploy_hostname = get_conf("DEPLOY_HOSTNAME");
if (!defined($Deploy_hostname)) {
    $Deploy_hostname = $Server_hostname;
}
my $Cosystem_hostname = get_conf("COSYSTEM_HOSTNAME");
if (!defined($Cosystem_hostname)) {
    $Cosystem_hostname = $Server_hostname;
}
if (is_conf("OAR_RUNTIME_DIRECTORY")) {
    OAR::Tools::set_default_oarexec_directory(get_conf("OAR_RUNTIME_DIRECTORY"));
}

#minimum of seconds to be considered like a hole in the gantt
my $Minimum_hole_time = 0;
if (is_conf("SCHEDULER_GANTT_HOLE_MINIMUM_TIME")) {
    $Minimum_hole_time = get_conf("SCHEDULER_GANTT_HOLE_MINIMUM_TIME");
}

# waiting time when a reservation has not all of its nodes
my $Reservation_waiting_timeout = 300;

# global variables: initialized in init_scheduler function
my %besteffort_resource_occupation;

# stock the job ids that where already send to almighty
my %To_launch_jobs_already_treated;

my $current_time_sec = 0;
my $current_time_sql = "0000-00-00 00:00:00";

my $resa_admin_waiting_timeout = get_conf("RESERVATION_WAITING_RESOURCES_TIMEOUT");

my $Walltime_change_enabled = uc(get_conf_with_default_param("WALLTIME_CHANGE_ENABLED", "NO"));

oar_info($Module_name, "Starting Meta Scheduler\n", $Session_id);
my @stats_start = OAR::IO::get_stats($base);

my %start_job_counts = OAR::IO::get_count_jobs_active_queues($base_ro);
my @job_count_logs = OAR::IO::format_count_jobs_active_queue(%start_job_counts);
foreach (@job_count_logs){
oar_info($Module_name,$_ . "\n",$Session_id);
}

###########################################################################################################
# Handle walltime change requests                                                                              #
###########################################################################################################

sub process_walltime_change_requests($) {
    my $dbh = shift;
    OAR::IO::lock_table($dbh, ['walltime_change']);
    my $jobs = OAR::IO::get_jobs_with_walltime_change($dbh);
    my $now  = OAR::IO::get_date($dbh);
    my $Walltime_change_apply_time =
      OAR::Conf::get_conf_with_default_param("WALLTIME_CHANGE_APPLY_TIME", 0);
    my $Walltime_increment = OAR::Walltime::get_default_increment;
    foreach my $jobid (keys(%$jobs)) {
        my $job       = $jobs->{$jobid};
        my $suspended = OAR::IO::get_job_suspended_sum_duration($dbh, $jobid, $now);
        my $fit       = $job->{pending};

        if ($fit > 0) {
            if ($Walltime_change_apply_time == 0) {
                my $last_walltime_request_event =
                  OAR::IO::get_last_event_from_type($dbh, "WALLTIME_REQUEST", $jobid);

                if ($job->{timeout} != 0 &&
                    ($now - $last_walltime_request_event->{date}) >= $job->{timeout}) {
                    my $message =
                      "Walltime request reached cancellation timeout, it is aborted (" .
                      OAR::IO::duration_to_sql($job->{timeout}) .
                      "), (pending: " . OAR::IO::duration_to_sql(0) . ", timeout: " .
                      OAR::IO::duration_to_sql($Walltime_change_cancellation_timeout) . ")";
                    oar_info($Module_name, $message, $Session_id, "$jobid");
                    OAR::IO::add_new_event($dbh, "WALLTIME_REQUEST_CANCELLATION", $jobid, $message);
                    OAR::IO::update_walltime_change_request($dbh, $jobid, 0, undef, undef, undef,
                        undef, undef, undef, undef, $Walltime_change_cancellation_timeout);
                    next;
                }
            }

            my $apply_time =
              OAR::Walltime::get_conf($Walltime_change_apply_time, $job->{queue_name},
                $job->{walltime} - $job->{granted}, 0);
            my $increment = OAR::Walltime::get_conf($Walltime_increment, $job->{queue_name},
                $job->{walltime} - $job->{granted}, 0);
            if (not defined($job->{force})) {
                $job->{force} = "NO";
            }
            oar_info(
                $Module_name,
                "walltime change: pending=" . $fit . "s delay_next_jobs=" .
                  $job->{delay_next_jobs} . " force=" . $job->{force} . " whole=" .
                  $job->{whole} . " apply_time=${apply_time}s increment=${increment}s\n",
                $Session_id,
                "$jobid");
            if ($job->{force} ne "YES") {
                my $delay = $job->{start_time} + $job->{walltime} + $suspended - $apply_time - $now;
                if ($apply_time > 0 and $delay > 0) {
                    oar_info($Module_name, "walltime change could apply in ${delay}s\n",
                        $Session_id, "$jobid");
                    next;    # next if activation time is set and activation date in the future
                }
                if ($increment > 0 and $increment < $fit) {
                    $fit = $increment;
                }
            }
            oar_info($Module_name, "walltime change try: $fit/" . $job->{pending} . "s\n",
                $Session_id, "$jobid");
            my $from  = $job->{start_time} + $job->{walltime} + $suspended;
            my $to    = $from + $fit;
            my $types = OAR::IO::get_job_types_hash($dbh, $jobid);
            if (exists($types->{inner})) {
                my $container_job = OAR::IO::get_running_job($dbh, $types->{inner});
                if (defined($container_job)) {
                    if ($container_job->{start_time} + $container_job->{moldable_walltime} < $to) {
                        $to =
                          $container_job->{start_time} +
                          $container_job->{moldable_walltime}
                          ;    #container should never be suspended, makes no sense
                        oar_info(
                            $Module_name,
                            "walltime change for inner job limited to the container's boundaries: "
                              . ($to - $from) . "s\n",
                            $Session_id,
                            "$jobid");
                    }
                } else {
                    oar_info($Module_name,
                        "walltime change for inner job but container is not found ?\n",
                        $Session_id, "$jobid");
                }
            }
            if (exists($types->{deadline})) {
                my $deadline = OAR::IO::sql_to_local($types->{deadline});
                if ($deadline < $to) {
                    $to = $deadline;
                    oar_info(
                        $Module_name,
                        "walltime change for job limited by its deadline to " .
                          ($to - $from) . "s\n",
                        $Session_id,
                        "$jobid");
                }
            }
            $fit =
              OAR::IO::get_possible_job_end_time_in_interval($dbh, $from, $to, $job->{resources},
                $Security_time_overhead, $job->{delay_next_jobs},
                $types, $job->{job_user}, $job->{job_name}) -
              $from;
            if ($fit <= 0) {
                oar_info(
                    $Module_name,
                    "walltime cannot be changed for now (pending: " .
                      OAR::IO::duration_to_sql_signed($job->{pending}) . ")\n",
                    $Session_id,
                    "$jobid");
                next;
            }
        } elsif ($fit < 0) {
            my $job_remaining_time = $job->{start_time} + $job->{walltime} + $suspended - $now;
            if ($job_remaining_time < -$fit) {
                $fit = -$job_remaining_time;
            }
        }

        my $new_walltime = $job->{walltime} + $fit;
        my $new_pending  = $job->{pending} - $fit;

        if (($job->{whole} eq "YES") && ($new_pending != 0)) {
            oar_info(
                $Module_name,
                "walltime cannot be changed to whole pending for now (pending: " .
                  OAR::IO::duration_to_sql_signed($job->{pending}) . ")\n",
                $Session_id,
                "$jobid");
            next;
        }

        my $message =
          "Walltime changed: " . OAR::IO::duration_to_sql($new_walltime) .
          " (granted: " . OAR::IO::duration_to_sql_signed($fit) .
          "/pending: " . OAR::IO::duration_to_sql_signed($new_pending) .
          (($job->{force} eq 'YES') ? "/force" : "") . (($job->{whole} eq 'YES') ? "/whole" : "") .
          (($job->{delay_next_jobs} eq 'YES') ? "/delay next jobs" : "") . (($new_pending == 0) ?
              "/timeout: " . OAR::IO::duration_to_sql($Walltime_change_cancellation_timeout) : "")
          .
          ")";
        oar_info($Module_name, "$message\n", $Session_id, "$jobid");
        OAR::IO::update_walltime_change_request(
            $dbh,
            $jobid,
            $new_pending,
            ($new_pending == 0) ? 'NO' : undef,
            ($new_pending == 0) ? 'NO' : undef,
            ($new_pending == 0) ? 'NO' : undef,
            $job->{granted} + $fit,
            ($job->{force} eq 'YES' and $fit > 0) ? ($job->{granted_with_force} + $fit) : undef,
            ($job->{delay_next_jobs} eq 'YES' and $fit > 0) ?
              ($job->{granted_with_delaying_next_jobs} + $fit) : undef,
            ($job->{whole} eq 'YES' and $fit > 0) ? ($job->{granted_with_whole} + $fit)   : undef,
            ($new_pending == 0)                   ? $Walltime_change_cancellation_timeout : undef);
        OAR::IO::change_walltime($dbh, $jobid, $new_walltime, $message);
    }
    OAR::IO::unlock_table($dbh);
}

# Walltime change disabled if set to -1
if ($Walltime_change_enabled eq "YES") {
    process_walltime_change_requests($base);
}

###########################################################################################################
# Initialize Gantt tables with scheduled reservation jobs, Running jobs, toLaunch jobs and Launching jobs #
###########################################################################################################

# Advance reservations in a near future (defined by the threshold) use the same
# resource order as the batch jobs, e.g. standby and besteffort resources are
# taken into account for resources selection. Otherwise the advance reservation
# specific resource order is used.
sub get_job_order_part($) {
    my $job   = shift;
    my $order = $Order_part_ar;
    if ($job->{start_time} - $current_time_sec < $Order_part_ar_threshold) {
        $order = $Order_part;
    }
    oar_info($Module_name, "use scheduling order: $order\n", $Session_id, "$job->{job_id}");
    return $order;
}

# First get resources that we must add for each reservation jobs
if (defined($Resources_to_always_add_type)) {
    my $tmp_result_state_resources =
      OAR::IO::get_specific_resource_states($base, $Resources_to_always_add_type);
    if (defined($tmp_result_state_resources->{"Alive"})) {
        @Resources_to_always_add = @{ $tmp_result_state_resources->{"Alive"} };
        oar_info(
            $Module_name,
            "Got resources to automatically add to every reservations: @Resources_to_always_add\n",
            $Session_id);
    }
}

$Reservation_waiting_timeout = $resa_admin_waiting_timeout
  if (defined($resa_admin_waiting_timeout));

# Take care of the currently (or nearly) running jobs
# Lock to prevent bipbip update in same time
OAR::IO::lock_table(
    $base,
    [   "jobs",                       "assigned_resources",
        "gantt_jobs_predictions",     "gantt_jobs_resources",
        "job_types",                  "moldable_job_descriptions",
        "resources",                  "job_state_logs",
        "gantt_jobs_predictions_log", "gantt_jobs_resources_log"
    ]);

#calculate now date with no overlap with other jobs
my $previous_ref_time_sec = OAR::IO::get_gantt_date($base);
$current_time_sec = OAR::IO::get_date($base);
if ($current_time_sec < $previous_ref_time_sec) {

    # The system is very fast!!!
    my $tmp_delta = $previous_ref_time_sec - $current_time_sec;
    if ($tmp_delta > 60) {
        oar_error(
            $Module_name,
            "init_scheduler: The previous scheduler step was performed $tmp_delta seconds in the future. It is bad, maybe the system clock of the DB server has changed. This can results in bad scheduling decisions.\n",
            $Session_id);
    } else {
        $current_time_sec = $previous_ref_time_sec;
        if ($tmp_delta > 1) {

            # Avoid race condition that could increment too much the current time indefinitely
            oar_info(
                $Module_name,
                "init_scheduler: The previous scheduler step was performed $tmp_delta seconds in the future. So we have to wait or the scheduling decisions will be bad\n",
                $Session_id);
            sleep($tmp_delta);
        }
    }
}
$current_time_sec++;
$current_time_sql = OAR::IO::local_to_sql($current_time_sec);

my ($Alive_resource_id_vec,     undef) = OAR::IO::get_resource_ids_in_state($base, "Alive");
my ($Absent_resource_id_vec,    undef) = OAR::IO::get_resource_ids_in_state($base, "Absent");
my ($Suspected_resource_id_vec, undef) = OAR::IO::get_resource_ids_in_state($base, "Suspected");
my ($Dead_resource_id_vec, @Dead_resource_ids) = OAR::IO::get_resource_ids_in_state($base, "Dead");

oar_info(
    $Module_name,
    "Retrieve information for already scheduled reservations from database before flush (keep assign resources)\n",
    $Session_id);
my $reservation_already_there = OAR::IO::get_waiting_reservations_already_scheduled($base);
OAR::IO::gantt_flush_tables($base, $reservation_already_there, $Log_scheduling);
OAR::IO::set_gantt_date($base, $current_time_sec);

#Init the gantt chart with all resources
oar_info($Module_name, "Initialize the gantt structure\n", $Session_id);
my ($Max_resources, $All_resource_list_vec, $Only_default_type_resource_vec) =
  OAR::IO::get_vecs_resources($base);
my $Gantt = {};
$Gantt->{0}->{""}->{""}->{""} =
  OAR::Schedulers::GanttHoleStorage::new($Max_resources, $Minimum_hole_time);
OAR::Schedulers::GanttHoleStorage::add_new_resources($Gantt->{0}->{""}->{""}->{""},
    $All_resource_list_vec);

oar_info($Module_name,
    "Begin processing of already scheduled reservations (accepted with resources assigned)\n",
    $Session_id);

# Add already scheduled reservations into the gantt
my @reservation_already_there_list = keys(%{$reservation_already_there});
while (@reservation_already_there_list) {
    my $i                = shift(@reservation_already_there_list);
    my $container_id     = 0;
    my $inner_id         = 0;
    my $placeholder_name = "";
    my $allowed_name     = "";
    my $timesharing_user = "";
    my $timesharing_name = "";
    my $vec              = '';
    my $types            = OAR::IO::get_job_types_hash($base, $i);

    if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)) {
        $inner_id = $1;
        if (defined($Gantt->{$inner_id}->{""}->{""}->{""})) {
            oar_info($Module_name, "inner job: using container $inner_id\n", $Session_id, "$i");
        } else {
            if (grep(/^$inner_id$/, @reservation_already_there_list)) {

                # The container job is later in the list, postpone inner job
                oar_info($Module_name,
                    "inner job: container $inner_id is not known yet, postponing job.\n",
                    $Session_id, "$i");
                push @reservation_already_there_list, $i;
                next;
            } else {
                oar_info(
                    $Module_name,
                    "inner job: container $inner_id does not exist but job is already scheduled, use container 0.\n",
                    $Session_id,
                    "$i");
                $inner_id = 0;
            }
        }
    }
    my $job_there = OAR::IO::get_job($base, $i);
    foreach my $r (@{ $reservation_already_there->{$i}->{resources} }) {
        vec($vec, $r, 1) = 1;
    }
    if (defined($types->{container}))
    {    # A container job cannot be placeholder or allowed or timesharing.
        oar_info($Module_name, "Job is ($inner_id,,,)\n", $Session_id, "$i");
        $container_id = $i;
        oar_info($Module_name, "container job: create gantt ($container_id,,,)\n",
            $Session_id, "$i");
        $Gantt->{$container_id}->{""}->{""}->{""} =
          OAR::Schedulers::GanttHoleStorage::new_with_1_hole(
            $Max_resources,
            $Minimum_hole_time,
            $reservation_already_there->{$i}->{start_time},
            $reservation_already_there->{$i}->{walltime} + $Security_time_overhead,
            $vec,
            $All_resource_list_vec);
    } else {
        ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) =
          OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder(
            $Gantt,
            $job_there->{job_user},
            $job_there->{job_name},
            $types, $inner_id, $Module_name, $Session_id, $i);
    }

    #Fill all other gantts
    OAR::Schedulers::GanttHoleStorage::fill_gantts(
        $Gantt, $reservation_already_there->{$i}->{start_time},
        $reservation_already_there->{$i}->{walltime} + $Security_time_overhead, $vec, $inner_id,
        $placeholder_name, $allowed_name, $timesharing_user, $timesharing_name, $Module_name,
        $Session_id,       $i);
}
oar_info($Module_name, "End processing of already scheduled reservations\n", $Session_id);

oar_info($Module_name, "Begin processing of current jobs\n", $Session_id);
my @initial_jobs = OAR::IO::get_jobs_in_multiple_states($base,
    [ "Running", "toLaunch", "Launching", "Finishing", "Suspended", "Resuming" ]);
while (@initial_jobs) {
    my $job              = shift(@initial_jobs);
    my $container_id     = 0;
    my $inner_id         = 0;
    my $placeholder_name = "";
    my $allowed_name     = "";
    my $timesharing_user = "";
    my $timesharing_name = "";
    my $i                = $job->{job_id};

    next if ($job->{assigned_moldable_job} == 0);

    my $types = OAR::IO::get_job_types_hash($base, $job->{job_id});

    if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)) {
        $inner_id = $1;
        if (defined($Gantt->{$inner_id}->{""}->{""}->{""})) {
            oar_info($Module_name, "inner job: using container $inner_id\n", $Session_id, "$i");
        } else {
            if (grep(/^$inner_id$/, map { $_->{job_id} } @initial_jobs)) {

                # The container job is later in the list, postpone inner job
                oar_info($Module_name,
                    "inner job: container $inner_id is not known yet, postponing job.\n",
                    $Session_id, "$i");
                push @initial_jobs, $job;
                next;
            } else {
                oar_info(
                    $Module_name,
                    "inner job: container $inner_id does not exist but job is running, use container 0.\n",
                    $Session_id,
                    "$i");
                $inner_id = 0;
            }
        }
    }

    my $mold = OAR::IO::get_current_moldable_job($base, $job->{assigned_moldable_job});

    # The list of resources on which the job is running
    my ($resource_list_vec, @resource_list) =
      OAR::IO::get_job_current_resources($base, $job->{assigned_moldable_job}, undef);

    my $date;
    if ($job->{start_time} == 0) {
        $date = $current_time_sec;
    } elsif ($job->{start_time} + $mold->{moldable_walltime} < $current_time_sec) {
        $date = $current_time_sec - $mold->{moldable_walltime};
    } else {
        $date = $job->{start_time};
    }
    oar_debug($Module_name, "add job in database\n", $Session_id, "$i");
    OAR::IO::add_gantt_scheduled_jobs($base, $job->{assigned_moldable_job}, $date, \@resource_list);

  # Ignore besteffort jobs (only handled in the besteffort queue => adv. resa cannot be bestfeffort)
    if (!defined($types->{besteffort})) {
        my $job_duration = $mold->{moldable_walltime};
        if ($job->{state} eq "Suspended") {

            # Remove resources of the type specified in SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
            ($resource_list_vec, @resource_list) = OAR::IO::get_job_current_resources(
                $base,
                $job->{assigned_moldable_job},
                \@Sched_available_suspended_resource_type);
        }
        if ($job->{suspended} eq "YES") {

            # This job was suspended so we must recalculate the walltime
            $job_duration +=
              OAR::IO::get_job_suspended_sum_duration($base, $job->{job_id}, $current_time_sec);
        }

        if (defined($types->{container}))
        {    # A container job cannot be placeholder or allowed or timesharing.
            oar_info($Module_name, "Job is ($inner_id,,,)\n", $Session_id, "$i");
            $container_id = $i;
            oar_info($Module_name, "container job: create gantt ($container_id,,,)\n",
                $Session_id, "$i");
            $Gantt->{$container_id}->{""}->{""}->{""} =
              OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources,
                $Minimum_hole_time, $date, $job_duration + $Security_time_overhead,
                $resource_list_vec, $All_resource_list_vec);
        } else {
            ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) =
              OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder(
                $Gantt,      $job->{job_user}, $job->{job_name}, $types, $inner_id, $Module_name,
                $Session_id, $i);
        }

        #Fill all other gantts
        OAR::Schedulers::GanttHoleStorage::fill_gantts(
            $Gantt,                                  $date,
            $job_duration + $Security_time_overhead, $resource_list_vec,
            $inner_id,                               $placeholder_name,
            $allowed_name,                           $timesharing_user,
            $timesharing_name,                       $Module_name,
            $Session_id,                             $i);
    } else {

        #Stock information about besteffort jobs
        foreach my $r (@resource_list) {
            $besteffort_resource_occupation{$r} = $job;
        }
    }
}
OAR::IO::unlock_table($base);
oar_info($Module_name, "End processing of current jobs\n", $Session_id);

# handle accepted advance reservations which do not have assigned resources, mostly because the prediction tables were completely flushed from outside of OAR
oar_info($Module_name,
    "Begin processing of the accepted reservations without assigned resources\n", $Session_id);
my @Rjobs = OAR::IO::get_waiting_reservation_jobs($base);
while (@Rjobs) {
    my $job = shift(@Rjobs);
    next if (defined($reservation_already_there->{ $job->{job_id} }));

    my $container_id     = 0;
    my $inner_id         = 0;
    my $placeholder_name = "";
    my $allowed_name     = "";
    my $timesharing_user = "";
    my $timesharing_name = "";
    my $i                = $job->{job_id};

    my $types = OAR::IO::get_job_types_hash($base, $i);
    if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)) {
        $inner_id = $1;
        if (defined($Gantt->{$inner_id}->{""}->{""}->{""})) {
            oar_info($Module_name, "inner job: using container $inner_id\n", $Session_id, "$i");
        } else {
            if (grep(/^$inner_id$/, map { $_->{job_id} } @Rjobs)) {

                # The container job is later in the list, postpone inner job
                oar_info($Module_name,
                    "inner job: container $inner_id is not known yet, postponing job.\n",
                    $Session_id, "$i");
                push @Rjobs, $job;
            } else {
                oar_info($Module_name, "inner job: container $inner_id does not exist.\n",
                    $Session_id, "$i");
                OAR::IO::set_job_message($base, $i, "Container $inner_id does not exist");
                OAR::IO::set_job_scheduler_info($base, $i, "Container $inner_id does not exist");
                $inner_id = 0;
            }
            next;
        }
    }

    my $job_descriptions = OAR::IO::get_resources_data_structure_current_job($base, $job->{job_id});

    # For reservation we take the first moldable job
    my $moldable = $job_descriptions->[0];

# Get the list of resources where the reservation will be able to be launched = resources in state Alive, Absent or Suspected
# It is assumed here that Absent and Suspected resources could become Alive by the time of the start of the advance reservation job
    my $available_resources_vector =
      $Alive_resource_id_vec | $Absent_resource_id_vec | $Suspected_resource_id_vec;
    if (defined($types->{container}))
    {    # A container job cannot be placeholder or allowed or timesharing.
        oar_info(
            $Module_name,
            "Job is ($inner_id,,,) and is a container (ignoring any timesharing or placeholder)\n",
            $Session_id,
            "$i");
    } else {
        ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) =
          OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder($Gantt,
            $job->{job_user}, $job->{job_name}, $types, $inner_id, $Module_name, $Session_id, $i);
    }

    my $free_resources_vec = OAR::Schedulers::GanttHoleStorage::get_free_resources(
        $Gantt->{$inner_id}->{$allowed_name}->{$timesharing_user}->{$timesharing_name},
        $job->{start_time}, $moldable->[1] + $Security_time_overhead,);
    $available_resources_vector &= $free_resources_vec;

    my $job_properties = "\'1\'";
    if ((defined($job->{properties})) and ($job->{properties} ne "")) {
        $job_properties = $job->{properties};
    }

    my $resource_id_used_list_vector = '';
    my @tree_list;
    foreach my $m (@{ $moldable->[0] }) {
        my $tmp_properties = "\'1\'";
        if ((defined($m->{property})) and ($m->{property} ne "")) {
            $tmp_properties = $m->{property};
        }
        my $tmp_tree;

        $tmp_tree =
          OAR::IO::get_possible_wanted_resources($base_ro, $available_resources_vector,
            $resource_id_used_list_vector, \@Dead_resource_ids,
            "$job_properties AND $tmp_properties",
            $m->{resources}, get_job_order_part($job));
        $tmp_tree =
          OAR::Schedulers::ResourceTree::delete_tree_nodes_with_not_enough_resources_and_unnecessary_subtrees(
            $tmp_tree, undef);
        push(@tree_list, $tmp_tree);
        my @leafs = OAR::Schedulers::ResourceTree::get_tree_leafs($tmp_tree);
        foreach my $l (@leafs) {
            vec($resource_id_used_list_vector,
                OAR::Schedulers::ResourceTree::get_current_resource_value($l), 1)
              = 1;
        }
    }

    my @resources;
    foreach my $t (@tree_list) {
        foreach my $r (OAR::Schedulers::ResourceTree::get_tree_leafs($t)) {
            push(@resources, OAR::Schedulers::ResourceTree::get_current_resource_value($r));
        }
    }

    if ($#resources >= 0) {

        # We can schedule the job
        my $vec = '';
        foreach my $r (@resources) {
            vec($vec, $r, 1) = 1;
        }

        # Create gantt for the new container
        if (defined($types->{container})) {
            $container_id = $i;
            oar_info($Module_name, "container job: create gantt ($container_id,,,)\n",
                $Session_id, "$i");
            $Gantt->{$container_id}->{""}->{""}->{""} =
              OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources,
                $Minimum_hole_time, $job->{start_time}, $moldable->[1] + $Security_time_overhead,
                $vec, $All_resource_list_vec);
        }

        #Fill all other gantts
        OAR::Schedulers::GanttHoleStorage::fill_gantts(
            $Gantt,                                   $job->{start_time},
            $moldable->[1] + $Security_time_overhead, $vec,
            $inner_id,                                $placeholder_name,
            $allowed_name,                            $timesharing_user,
            $timesharing_name,                        $Module_name,
            $Session_id,                              $i);

        # Update database
        push(@resources, @Resources_to_always_add);
        oar_debug($Module_name, "add job in database\n", $Session_id, "$i");
        oar_warn($Module_name, "resources for reservation were lost, got them back\n",
            $Session_id, "$i");
        OAR::IO::add_gantt_scheduled_jobs($base, $moldable->[2], $job->{start_time}, \@resources);
        OAR::IO::add_new_event($base, "GET_RESOURCES_BACK_FOR_RESERVATION",
            $job->{job_id}, "Resources for reservation were lost, got them back");
        OAR::IO::set_job_message($base, $job->{job_id}, "Renewed resources for reservation");
    } else {
        oar_warn($Module_name, "Resources for reservation are lost, could not get them back\n",
            $Session_id, "$i");
        my $message = "No resource for reservation";
        if ($job->{message} ne $message) {

   # Sounds like it's the first time we notice that the reservation lost its resources, adding event
            OAR::IO::add_new_event($base, "UNABLE_TO_GET_RESOURCE_BACK_FOR_RESERVATION",
                $job->{job_id}, "Resources for reservation are lost, could not get them back");
        }
        OAR::IO::set_job_message($base, $job->{job_id}, $message);
    }
}
oar_info($Module_name, "End processing of the accepted reservations without assigned resources\n",
    $Session_id);

my @stats_end1p = OAR::IO::get_stats($base);
my $stats_end1p_str = OAR::IO::format_stats(@stats_start, @stats_end1p);
oar_info($Module_name, "End of first phase (Gantt initialization) ($stats_end1p_str)\n", $Session_id);

###########################################################################################################
# End init scheduler                                                                                      #
###########################################################################################################

# launch right reservation jobs
# arg: queue name
# return 1 if there is at least a job to treate, 2 if besteffort jobs must die
sub treate_waiting_reservation_jobs($) {
    my $queue_name = shift;

    oar_info($Module_name, "Queue $queue_name: begin processing of waiting reservations\n",
        $Session_id);

    my $return = 0;

    my @arrayJobs = OAR::IO::get_waiting_reservation_jobs_specific_queue($base, $queue_name);

    # See if there are reserved jobs to launch
    foreach my $job (@arrayJobs) {
        my $job_descriptions =
          OAR::IO::get_resources_data_structure_current_job($base, $job->{job_id});
        my $moldable = $job_descriptions->[0];

        # Test if the job is in the past
        if ($current_time_sec > $job->{start_time} + $moldable->[1]) {
            oar_warn($Module_name,
                "set job state to Error: reservation expired and couldn't be started\n",
                $Session_id, "$job->{job_id}");
            OAR::IO::set_job_state($base, $job->{job_id}, "Error");
            OAR::IO::set_job_message($base, $job->{job_id},
                "Reservation expired and couldn't be started.");
            $return = 1;
        }
        my @resa_alive_resources;
        my $jobtypes = OAR::IO::get_job_types_hash($base, $job->{job_id});

        if (defined($jobtypes->{state}) and
            $jobtypes->{state} eq "permissive" and
            (defined($jobtypes->{noop}) or defined($jobtypes->{cosystem}))) {
            oar_warn(
                $Module_name,
                "cosystem/noop advance reservation with permissive resources state: not checking resources aliveness\n",
                $Session_id,
                "$job->{job_id}");
        } else {
            my $standbystart = (
                (defined($jobtypes->{deploy})     and $jobtypes->{deploy} eq "standby")   or
                  (defined($jobtypes->{cosystem}) and $jobtypes->{cosystem} eq "standby") or
                  (defined($jobtypes->{noop})     and $jobtypes->{noop} eq "standby"));
            if ($standbystart) {
                oar_info($Module_name,
                    "reservation is allowed to start with some resources in standby\n",
                    $Session_id, "$job->{job_id}");
                @resa_alive_resources =
                  OAR::IO::get_gantt_Alive_or_Standby_resources_for_job($base, $moldable->[2],
                    $job->{start_time} + $moldable->[1] + $Security_time_overhead);
            } else {
                @resa_alive_resources =
                  OAR::IO::get_gantt_Alive_resources_for_job($base, $moldable->[2]);
            }

            # test if the job is going to be launched and there is no Alive node
            if (($#resa_alive_resources < 0) && ($job->{start_time} <= $current_time_sec)) {
                oar_warn($Module_name, "reservation is waiting because no resource is available\n",
                    $Session_id, "$job->{job_id}");
                OAR::IO::set_gantt_job_startTime($base, $moldable->[2], $current_time_sec + 1);
            } elsif ($job->{start_time} <= $current_time_sec) {
                my @resa_resources = OAR::IO::get_gantt_resources_for_job($base, $moldable->[2]);
                if ($job->{start_time} + $Reservation_waiting_timeout > $current_time_sec) {
                    if ($#resa_resources > $#resa_alive_resources) {

             # we have not the same number of nodes than in the query --> wait the specified timeout
                        oar_warn(
                            $Module_name,
                            "reservation is waiting because not all resources are available yet (timeout in "
                              .
                              ( $job->{start_time} +
                                  $Reservation_waiting_timeout -
                                  $current_time_sec
                              ) .
                              " seconds)\n",
                            $Session_id,
                            "$job->{job_id}");
                        OAR::IO::set_gantt_job_startTime($base, $moldable->[2],
                            ($current_time_sec + 1));
                    }
                } else {

       #Check if resources are in Alive state otherwise remove them, the job is going to be launched
                    foreach my $r (@resa_resources) {
                        my $resource_info = OAR::IO::get_resource_info($base, $r);
                        if (
                            not($resource_info->{state} eq "Alive" or
                                (   $standbystart and
                                    $resource_info->{state} eq "Absent" and
                                    ($job->{start_time} + $moldable->[1] + $Security_time_overhead)
                                    < $resource_info->{available_upto}))
                        ) {
                            oar_warn(
                                $Module_name,
                                "drop resource $r of the reservation, because resource is $resource_info->{state})\n",
                                $Session_id,
                                "$job->{job_id}");
                            OAR::IO::remove_gantt_resource_job($base, $moldable->[2], $r);
                        }
                    }
                    if ($#resa_resources > $#resa_alive_resources) {
                        OAR::IO::add_new_event(
                            $base,
                            "SCHEDULER_REDUCE_NB_NODES_FOR_RESERVATION",
                            $job->{job_id},
                            "[MetaSched] Reduce the number of resources for the job $job->{job_id}."
                        );
                        my $n = $#resa_alive_resources + 1;
                        if ($job->{message} =~ s/R\=\d+/R\=$n/g) {
                            OAR::IO::set_job_message($base, $job->{job_id}, $job->{message});
                        }
                    }
                }
            }
        }
    }
    oar_info($Module_name, "Queue $queue_name: end processing of waiting reservations\n",
        $Session_id);
    return ($return);
}

# check for jobs with reservation
# arg: queue name
# return 1 if there is at least a job to treate else 0
sub check_reservation_jobs($) {
    my $queue_name = shift;

    oar_info($Module_name, "Queue $queue_name: begin processing of new reservations\n",
        $Session_id);

    my $return = 0;

    #Init the gantt chart with all resources
    my $Gantt = {};
    $Gantt->{0}->{""}->{""}->{""} =
      OAR::Schedulers::GanttHoleStorage::new($Max_resources, $Minimum_hole_time);
    OAR::Schedulers::GanttHoleStorage::add_new_resources($Gantt->{0}->{""}->{""}->{""},
        $All_resource_list_vec);

    # Find jobs to check
    my @jobs_to_sched =
      OAR::IO::get_waiting_toSchedule_reservation_jobs_specific_queue($base, $queue_name);
    if ($#jobs_to_sched >= 0) {
        oar_info($Module_name,
            "Building the gantt structure and filling in already scheduled jobs\n", $Session_id);

      # Build gantt diagram of other jobs
      # Take care of currently scheduled jobs except besteffort jobs if queue_name is not besteffort
        my ($order, %already_scheduled_jobs) = OAR::IO::get_gantt_scheduled_jobs($base);
        my @already_scheduled_jobs_list = @{$order};
        while (@already_scheduled_jobs_list) {
            my $i                = shift(@already_scheduled_jobs_list);
            my $container_id     = 0;
            my $inner_id         = 0;
            my $placeholder_name = "";
            my $allowed_name     = "";
            my $timesharing_user = "";
            my $timesharing_name = "";
            my $types            = OAR::IO::get_job_types_hash($base, $i);

            # Ignore besteffort jobs
            if ((!defined($types->{besteffort})) or ($queue_name eq "besteffort")) {
                my @resource_list     = @{ $already_scheduled_jobs{$i}->[3] };
                my $resource_list_vec = $already_scheduled_jobs{$i}->[10];
                my $job_duration      = $already_scheduled_jobs{$i}->[1];
                if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)) {
                    $inner_id = $1;
                    if (defined($Gantt->{$inner_id}->{""}->{""}->{""})) {
                        oar_info($Module_name, "inner job: using container $inner_id\n",
                            $Session_id, "$i");
                    } else {
                        if (grep(/^$inner_id$/, @already_scheduled_jobs_list)) {

                            # The container job is later in the list, postpone inner job
                            oar_info(
                                $Module_name,
                                "inner job: container $inner_id is not known yet, postponing job.\n",
                                $Session_id,
                                "$i");
                            push @already_scheduled_jobs_list, $i;
                            next;
                        } else {
                            oar_info(
                                $Module_name,
                                "inner job: container $inner_id does not exist but job is already scheduled, use container 0.\n",
                                $Session_id,
                                "$i");
                            $inner_id = 0;
                        }
                    }
                }
                if ($already_scheduled_jobs{$i}->[4] eq "Suspended") {

             # Remove resources of the type specified in SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
                    ($resource_list_vec, @resource_list) = OAR::IO::get_job_current_resources(
                        $base,
                        $already_scheduled_jobs{$i}->[7],
                        \@Sched_available_suspended_resource_type);
                    next if ($#resource_list < 0);
                }
                if ($already_scheduled_jobs{$i}->[8] eq "YES") {

                    # This job was suspended so we must recalculate the walltime
                    $job_duration +=
                      OAR::IO::get_job_suspended_sum_duration($base, $i, $current_time_sec);
                }

                if (defined($types->{container}))
                {    # A container job cannot be placeholder or allowed or timesharing.
                    oar_info($Module_name, "Job is ($inner_id,,,)\n", $Session_id, "$i");
                    $container_id = $i;
                    oar_info($Module_name, "container job: create gantt ($container_id,,,)\n",
                        $Session_id, "$i");
                    $Gantt->{$container_id}->{""}->{""}->{""} =
                      OAR::Schedulers::GanttHoleStorage::new_with_1_hole(
                        $Max_resources,                   $Minimum_hole_time,
                        $already_scheduled_jobs{$i}->[0], $job_duration + $Security_time_overhead,
                        $resource_list_vec,               $All_resource_list_vec);
                } else {
                    ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) =
                      OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder(
                        $Gantt,
                        $already_scheduled_jobs{$i}->[5],
                        $already_scheduled_jobs{$i}->[6],
                        $types, $inner_id, $Module_name, $Session_id, $i);
                }

                #Fill all other gantts
                OAR::Schedulers::GanttHoleStorage::fill_gantts(
                    $Gantt,                                  $already_scheduled_jobs{$i}->[0],
                    $job_duration + $Security_time_overhead, $resource_list_vec,
                    $inner_id,                               $placeholder_name,
                    $allowed_name,                           $timesharing_user,
                    $timesharing_name,                       $Module_name,
                    $Session_id,                             $i);
            }
        }
        oar_info($Module_name, "Try and schedule new reservations of queue $queue_name\n",
            $Session_id);
        while (@jobs_to_sched) {
            my $job              = shift(@jobs_to_sched);
            my $container_id     = 0;
            my $inner_id         = 0;
            my $placeholder_name = "";
            my $allowed_name     = "";
            my $timesharing_user = "";
            my $timesharing_name = "";
            my $i                = $job->{job_id};
            my $types            = OAR::IO::get_job_types_hash($base, $i);
            my $job_descriptions =
              OAR::IO::get_resources_data_structure_current_job($base, $job->{job_id});

            # It is a reservation, we take care only of the first moldable job
            my $moldable = $job_descriptions->[0];
            my $duration = $moldable->[1];

            #look if reservation is too old
            if ($current_time_sec >= ($job->{start_time} + $duration)) {
                oar_warn($Module_name, "Canceling job: reservation is too old\n", $Session_id,
                    "$i");
                OAR::IO::set_job_message($base, $job->{job_id}, "Reservation too old");
                OAR::IO::set_job_state($base, $job->{job_id}, "toError");
            } else {
                if ($job->{start_time} < $current_time_sec) {
                    $job->{start_time} = $current_time_sec;

                    #OAR::IO::set_running_date_arbitrary($base,$job->{job_id},$current_time_sql);
                }

# Get the list of resources where the reservation will be able to be launched = resources in state Alive, Absent or Suspected
# It is assumed here that Absent and Suspected resources could become Alive by the time of the start of the advance reservation job
                my $available_resources_vector =
                  $Alive_resource_id_vec | $Absent_resource_id_vec | $Suspected_resource_id_vec;
                if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)) {
                    $inner_id = $1;
                    if ($inner_id == $i) {
                        my $message =
                          "inner job: job cannot be its own container, setting it to error";
                        oar_warn($Module_name, "$message\n", $Session_id, "$i");
                        OAR::IO::set_job_message($base, $i, ucfirst($message));
                        OAR::IO::set_job_state($base, $i, "toError");
                        next;
                    }
                    if (defined($Gantt->{$inner_id}->{""}->{""}->{""})) {
                        oar_info($Module_name, "inner job: using container $inner_id\n",
                            $Session_id, "$i");
                    } else {

                        #is the container yet to schedule, by this scheduler or another ?
                        if (grep(/^$inner_id$/, map { $_->{job_id} } @jobs_to_sched)) {

                            # The container job is later in the list, postpone inner job
                            oar_info(
                                $Module_name,
                                "inner job: container $inner_id is not known yet, postponing job.\n",
                                $Session_id,
                                "$i");
                            push @jobs_to_sched, $job;
                        } elsif (grep(/^$inner_id$/, OAR::IO::get_all_waiting_jobids($base))) {
                            my $message =
                              "inner job: container $inner_id is yet to scheduled, inner job not scheduled";
                            oar_info($Module_name, "$message\n", $Session_id, "$i");
                            OAR::IO::set_job_message($base, $i, ucfirst($message));
                            OAR::IO::set_job_scheduler_info($base, $i, $message);
                        } else {
                            my $message =
                              "inner job: container $inner_id does not exist, inner job will never run, setting it to error";
                            oar_warn($Module_name, "$message\n", $Session_id, "$i");
                            OAR::IO::set_job_message($base, $i, ucfirst($message));
                            OAR::IO::set_job_state($base, $i, "toError");
                        }
                        next;
                    }
                }
                if (defined($types->{container}))
                {    # A container job cannot be placeholder or allowed or timesharing.
                    oar_info(
                        $Module_name,
                        "Job is ($inner_id,,,) and is a container (ignoring any timesharing or placeholder)\n",
                        $Session_id,
                        "$i");
                } else {
                    ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) =
                      OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder(
                        $Gantt,       $job->{job_user}, $job->{job_name}, $types, $inner_id,
                        $Module_name, $Session_id,      $i);
                }

                my $job_properties = "\'1\'";
                if ((defined($job->{properties})) and ($job->{properties} ne "")) {
                    $job_properties = $job->{properties};
                }

                #my $resource_id_used_list_vector = '';
                my @tree_list;
                foreach my $m (@{ $moldable->[0] }) {
                    my $tmp_properties = "\'1\'";
                    if ((defined($m->{property})) and ($m->{property} ne "")) {
                        $tmp_properties = $m->{property};
                    }
                    my $tmp_tree =
                      OAR::IO::get_possible_wanted_resources($base_ro, $available_resources_vector,
                        undef, \@Dead_resource_ids, "$job_properties AND $tmp_properties",
                        $m->{resources}, get_job_order_part($job));
                    $tmp_tree =
                      OAR::Schedulers::ResourceTree::delete_tree_nodes_with_not_enough_resources(
                        $tmp_tree);
                    push(@tree_list, $tmp_tree);
                }

                oar_info(
                    $Module_name,
                    "Find first hole with gantt($inner_id,$allowed_name,$timesharing_user,$timesharing_name)\n",
                    $Session_id,
                    "$i");
                my @hole = OAR::Schedulers::GanttHoleStorage::find_first_hole(
                    $Gantt->{$inner_id}->{$allowed_name}->{$timesharing_user}->{$timesharing_name},
                    $job->{start_time}, $duration + $Security_time_overhead,
                    \@tree_list,        30);
                if ($hole[0] == $job->{start_time}) {

                    # The reservation can be scheduled
                    my @resources;
                    foreach my $t (@{ $hole[1] }) {
                        foreach my $r (OAR::Schedulers::ResourceTree::get_tree_leafs($t)) {
                            push(@resources,
                                OAR::Schedulers::ResourceTree::get_current_resource_value($r));
                        }
                    }

                    # We can schedule the job
                    oar_warn($Module_name, "advance reservation is validated\n", $Session_id, "$i");
                    my $vec = '';
                    foreach my $r (@resources) {
                        vec($vec, $r, 1) = 1;
                    }

                    # Create gantt for the new container
                    if (defined($types->{container})) {
                        $container_id = $i;
                        oar_info($Module_name, "container job: create gantt ($container_id,,,)\n",
                            $Session_id, "$i");
                        $Gantt->{$container_id}->{""}->{""}->{""} =
                          OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources,
                            $Minimum_hole_time, $job->{start_time},
                            $duration + $Security_time_overhead,
                            $vec, $All_resource_list_vec);
                    }

                    #Fill all other gantts
                    OAR::Schedulers::GanttHoleStorage::fill_gantts(
                        $Gantt,                              $job->{start_time},
                        $duration + $Security_time_overhead, $vec,
                        $inner_id,                           $placeholder_name,
                        $allowed_name,                       $timesharing_user,
                        $timesharing_name,                   $Module_name,
                        $Session_id,                         $i);

                    # Update database
                    push(@resources, @Resources_to_always_add);
                    oar_debug($Module_name, "add job in database\n", $Session_id, "$i");
                    OAR::IO::add_gantt_scheduled_jobs($base, $moldable->[2], $job->{start_time},
                        \@resources);
                    OAR::IO::set_job_state($base, $job->{job_id}, "toAckReservation");
                } else {
                    oar_warn($Module_name,
                        "advance reservation cannot be validated, not enough resources\n",
                        $Session_id, "$i");
                    OAR::IO::set_job_state($base, $job->{job_id}, "toError");
                    if ($hole[0] == OAR::Schedulers::GanttHoleStorage::get_infinity_value()) {
                        OAR::IO::set_job_message($base, $job->{job_id},
                            "This reservation cannot run");
                    } else {
                        OAR::IO::set_job_message($base, $job->{job_id},
                            "This reservation could run at " . OAR::IO::local_to_sql($hole[0]));
                    }
                }
            }
            OAR::IO::set_job_resa_state($base, $job->{job_id}, "Scheduled");
            $return = 1;
        }
    }
    oar_info($Module_name, "Queue $queue_name: end processing of new reservations\n", $Session_id);
    return ($return);
}

# Detect if there are besteffort jobs to kill
# return 1 if there is at least 1 job to frag otherwise 0
sub check_jobs_to_kill() {
    oar_info($Module_name, "Begin processing of besteffort jobs to kill\n", $Session_id);
    my $return = 0;

    # Delete besteffort jobs in advance to let enough time to get back the nodes
    # in a good shape for the reservations
    my $besteffort_kill_duration =
      get_conf_with_default_param("SCHEDULER_BESTEFFORT_KILL_DURATION_BEFORE_RESERVATION", 0);
    $besteffort_kill_duration = $Security_time_overhead
      if ($besteffort_kill_duration < $Security_time_overhead);
    my %nodes_for_jobs_to_launch = OAR::IO::get_gantt_resources_for_jobs_to_launch($base,
        $current_time_sec + $besteffort_kill_duration);
    my %fragged_jobs = ();
    foreach my $r (keys(%nodes_for_jobs_to_launch)) {
        if (defined($besteffort_resource_occupation{$r})) {
            if (
                OAR::IO::is_timesharing_for_2_jobs(
                    $base, $nodes_for_jobs_to_launch{$r},
                    $besteffort_resource_occupation{$r}->{job_id})
            ) {
                oar_info(
                    $Module_name,
                    "Resource $r is needed for job $nodes_for_jobs_to_launch{$r}, but besteffort job $besteffort_resource_occupation{$r}->{job_id} can live, because timesharing compatible\n",
                    $Session_id,
                    $besteffort_resource_occupation{$r}->{job_id});
            } else {
                unless (defined($fragged_jobs{ $besteffort_resource_occupation{$r}->{job_id} })) {
                    my $skip_kill = 0;

                    # Check if we must checkpoint the besteffort job
                    if ($besteffort_resource_occupation{$r}->{checkpoint} > 0) {
                        my $checkpoint_first_date;
                        foreach my $e (
                            OAR::IO::get_job_events(
                                $base, $besteffort_resource_occupation{$r}->{job_id})
                        ) {
                            if ($e->{type} eq "CHECKPOINT") {
                                if ((!defined($checkpoint_first_date)) or
                                    ($checkpoint_first_date > $e->{date})) {
                                    $checkpoint_first_date = $e->{date};
                                }
                            }
                        }
                        if ((!defined($checkpoint_first_date)) or
                            (
                                $current_time_sec <= (
                                    $checkpoint_first_date +
                                      $besteffort_resource_occupation{$r}->{checkpoint}))
                        ) {
                            # Retrieve node names used by the job
                            my @hosts = OAR::IO::get_job_current_hostnames($base,
                                $besteffort_resource_occupation{$r}->{job_id});
                            my $types = OAR::IO::get_job_types_hash($base,
                                $besteffort_resource_occupation{$r}->{job_id});
                            if ($#hosts < 0 and
                                not defined($types->{cosystem}) and
                                not defined
                                ($types->{deploy})) {
                                oar_info(
                                    $Module_name,
                                    "Not sending checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id} (job with no host)\n",
                                    $Session_id);
                            } else {
                                my $host_to_connect = $hosts[0];
                                if (defined($types->{cosystem})) {
                                    $host_to_connect = $Cosystem_hostname;
                                } elsif (defined($types->{deploy})) {
                                    $host_to_connect = $Deploy_hostname;
                                }
                                oar_info(
                                    $Module_name,
                                    "Sending checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id} on host $host_to_connect\n",
                                    $Session_id,
                                    $besteffort_resource_occupation{$r}->{job_id});
                                OAR::IO::add_new_event(
                                    $base,
                                    "CHECKPOINT",
                                    $besteffort_resource_occupation{$r}->{job_id},
                                    "User oar (metasched) requested a checkpoint on the job $besteffort_resource_occupation{$r}->{job_id} on $host_to_connect"
                                );
                                $skip_kill = 1;
                                my $str_comment;
                                my $str_log;
                                my @exit_codes;

                                # Timeout the ssh command
                                eval {
                                    $SIG{ALRM} = sub { die "alarm\n" };
                                    alarm(OAR::Tools::get_ssh_timeout());
                                    @exit_codes =
                                      OAR::Tools::signal_oarexec($host_to_connect,
                                        $besteffort_resource_occupation{$r}->{job_id},
                                        "SIGUSR2", 1, $base, $Openssh_cmd, '');
                                    alarm(0);
                                };
                                if ($@) {
                                    if ($@ eq "alarm\n") {
                                        $str_log =
                                            "Cannot contact $host_to_connect, operation timed out ("
                                          . OAR::Tools::get_ssh_timeout()
                                          . " s). So I cannot send checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id}";
                                        $str_comment = "[$Module_name] $str_log";
                                        oar_warn($Module_name, "$str_log\n", $Session_id,
                                            $besteffort_resource_occupation{$r}->{job_id});
                                        OAR::IO::add_new_event($base, "CHECKPOINT_ERROR",
                                            $besteffort_resource_occupation{$r}->{job_id},
                                            $str_comment);
                                    } else {
                                        $str_log =
                                          "An unknown error occured during the sending of the checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id} on the host $host_to_connect";
                                        $str_comment = "[$Module_name] $str_log";
                                        oar_warn($Module_name, "$str_log\n", $Session_id,
                                            $besteffort_resource_occupation{$r}->{job_id});
                                        OAR::IO::add_new_event($base, "CHECKPOINT_ERROR",
                                            $besteffort_resource_occupation{$r}->{job_id},
                                            $str_comment);
                                    }
                                } else {
                                    if ($exit_codes[0] == 0) {
                                        $str_log =
                                          "The job $besteffort_resource_occupation{$r}->{job_id} was notified to checkpoint itself on the node $host_to_connect";
                                        $str_comment = "[$Module_name] $str_log";
                                        oar_info($Module_name, "$str_log\n", $Session_id,
                                            $besteffort_resource_occupation{$r}->{job_id});
                                        OAR::IO::add_new_event($base, "CHECKPOINT_SUCCESSFULL",
                                            $besteffort_resource_occupation{$r}->{job_id},
                                            $str_comment);
                                    } else {
                                        $str_log =
                                          "The kill command return a bad exit code (@exit_codes) for the job $besteffort_resource_occupation{$r}->{job_id} on the node $host_to_connect";
                                        $str_comment = "[$Module_name] $str_log";
                                        oar_warn($Module_name, "$str_log\n", $Session_id,
                                            $besteffort_resource_occupation{$r}->{job_id});
                                        OAR::IO::add_new_event($base, "CHECKPOINT_ERROR",
                                            $besteffort_resource_occupation{$r}->{job_id},
                                            $str_comment);
                                    }
                                }
                            }
                        } else {

                            # Retrieve node names used by the job
                            my @hosts = OAR::IO::get_job_current_hostnames($base,
                                $besteffort_resource_occupation{$r}->{job_id});
                            my $types = OAR::IO::get_job_types_hash($base,
                                $besteffort_resource_occupation{$r}->{job_id});
                            if ($#hosts >= 0 or
                                defined($types->{cosystem}) or
                                defined($types->{deploy})) {
                                my $str_log =
                                  "Checkpoint timeout for job $besteffort_resource_occupation{$r}->{job_id} ($current_time_sec > ($checkpoint_first_date+$besteffort_resource_occupation{$r}->{checkpoint})";
                                my $str_comment = "[$Module_name] $str_log";
                                oar_info($Module_name, "$str_log\n", $Session_id,
                                    $besteffort_resource_occupation{$r}->{job_id});
                                OAR::IO::add_new_event($base, "CHECKPOINT_TIMEOUT",
                                    $besteffort_resource_occupation{$r}->{job_id}, $str_comment);
                            }
                        }
                    }
                    if ($skip_kill == 0) {
                        oar_info(
                            $Module_name,
                            "Resource $r need to be freed for job $nodes_for_jobs_to_launch{$r}: killing besteffort job $besteffort_resource_occupation{$r}->{job_id}\n",
                            $Session_id,
                            $besteffort_resource_occupation{$r}->{job_id});
                        OAR::IO::add_new_event(
                            $base,
                            "BESTEFFORT_KILL",
                            $besteffort_resource_occupation{$r}->{job_id},
                            "[MetaSched] kill the besteffort job $besteffort_resource_occupation{$r}->{job_id}"
                        );
                        OAR::IO::lock_table($base, [ "frag_jobs", "event_logs", "jobs" ]);
                        OAR::IO::frag_job($base, $besteffort_resource_occupation{$r}->{job_id});
                        OAR::IO::unlock_table($base);
                    }
                    $fragged_jobs{ $besteffort_resource_occupation{$r}->{job_id} } = 1;
                    $return = 1;
                }
            }
        }
    }
    oar_info($Module_name, "End precessing of besteffort jobs to kill\n", $Session_id);
    return ($return);
}

# Tell Almighty to run a job
sub notify_to_run_job($$) {
    my ($base, $job_id) = @_;

    if (!defined($To_launch_jobs_already_treated{$job_id})) {
        if (OAR::IO::is_job_desktop_computing($base, $job_id)) {
            oar_info($Module_name, "Desktop computing job, not handling it!\n",
                $Session_id, $job_id);
        } else {
            my $err =
              OAR::Tools::notify_tcp_socket($Remote_host, $Remote_port, "OARRUNJOB_$job_id");
            if (!defined($err)) {
                $To_launch_jobs_already_treated{$job_id} = 1;
                oar_info($Module_name, "Notify almighty to launch the job\n", $Session_id, $job_id);
            } else {
                oar_warn($Module_name, "Not able to notify almighty to launch the job: $err\n",
                    $Session_id, $job_id);
            }
        }
    }
}

# Prepare a job to be run by bipbip
sub prepare_job_to_be_launched($$$$$) {
    my ($base, $job_id, $moldable_job_id, $job_submission_time, $resources_array_ref) = @_;

    my $running_date = $current_time_sec;
    if ($running_date < $job_submission_time) {
        $running_date = $job_submission_time;
    }
    OAR::IO::set_running_date_arbitrary($base, $job_id, $running_date);
    OAR::IO::set_assigned_moldable_job($base, $job_id, $moldable_job_id);

    my $insert_from_file = get_conf_with_default_param("INSERTS_FROM_FILE", "no");
    if ($insert_from_file eq 'yes') {
        OAR::IO::add_resource_job_pairs_from_file($base, $moldable_job_id, $resources_array_ref);
    } else {
        OAR::IO::add_resource_job_pairs($base, $moldable_job_id, $resources_array_ref);
    }
    OAR::IO::set_job_state($base, $job_id, "toLaunch");
    notify_to_run_job($base, $job_id);
}

# Detect if there are jobs to launch
# return 1 if there is at least 1 job to launch otherwise 0
sub check_jobs_to_launch() {
    oar_info($Module_name,
        "Begin processing of jobs to launch (start time <= $current_time_sql)\n", $Session_id);
    my $return_code    = 0;
    my %jobs_to_launch = OAR::IO::get_gantt_jobs_to_launch($base, $current_time_sec);
    foreach my $i (keys(%jobs_to_launch)) {
        oar_info($Module_name, "Set job state to toLaunch\n", $Session_id, "$i");

        # We must look at reservations to not go after the initial stop time
        my $mold = OAR::IO::get_current_moldable_job($base, $jobs_to_launch{$i}->[0]);
        my $job  = OAR::IO::get_job($base, $i);
        if (($job->{reservation} eq "Scheduled") and ($job->{start_time} < $current_time_sec)) {
            my $new_walltime =
              $mold->{moldable_walltime} - ($current_time_sec - $job->{start_time});
            my $new_walltime_sql  = OAR::IO::duration_to_sql($new_walltime);
            my $old_walltime_sql  = OAR::IO::duration_to_sql($mold->{moldable_walltime});
            my $old_starttime_sql = OAR::IO::local_to_sql($job->{start_time});
            OAR::IO::set_moldable_job_walltime($base, $jobs_to_launch{$i}->[0], $new_walltime);
            OAR::IO::set_gantt_job_startTime($base, $jobs_to_launch{$i}->[0], $current_time_sec);
            oar_info($Module_name,
                "Update job start time to $current_time_sql (was: $old_starttime_sql)\n",
                $Session_id, "$i");
            oar_info($Module_name,
                "Reduce job walltime to $new_walltime_sql (was: $old_walltime_sql)\n",
                $Session_id, "$i");
            OAR::IO::add_new_event($base, "REDUCE_RESERVATION_WALLTIME", $i,
                "Set actual start time (was: $old_starttime_sql) and walltime (was: $old_walltime_sql)"
            );

            if ($job->{message} =~ s/W\=\d+\:\d+\:\d+/W\=$new_walltime_sql/g) {
                OAR::IO::set_job_message($base, $i, $job->{message});
            }
        }
        prepare_job_to_be_launched(
            $base, $i,
            $jobs_to_launch{$i}->[0],
            $job->{submission_time},
            $jobs_to_launch{$i}->[1]);
        $return_code = 1;
    }

    oar_info($Module_name, "End processing of jobs to launch\n", $Session_id);
    return ($return_code);
}

my %initial_time = (
    "sec" => $current_time_sec,
    "sql" => $current_time_sql);

my @queues = OAR::IO::get_active_queues($base);
my $name;
my $policy;
foreach my $queue (@queues) {
    $name   = $queue->[0];
    $policy = $queue->[1];
    my $waiting_jobs = OAR::IO::is_waiting_job_specific_queue_present($base, $name);
    if ($waiting_jobs == 1) {
        my @stats_queue_start = OAR::IO::get_stats($base);
        oar_info($Module_name,
            "Queue $name: Launching scheduler $policy at time $initial_time{sql}\n", $Session_id);
        my ($sched_exit_code, $sched_signal_num, $sched_dumped_core) = (-1, -1, -1);
        my $sched_pid =
          open(READSCHED,
            "$binpath/schedulers/$policy $name $initial_time{sec} \"$initial_time{sql}\" |") or
          oar_warn(
            $Module_name,
            "Cannot run: \"$binpath/schedulers/$policy $name $initial_time{sec} \"$initial_time{sql}\"\"; $!\n",
            $Session_id);
        if (defined($sched_pid)) {
            while (my $l = <READSCHED>) {
                chop($l);
                oar_info($Module_name, "Read on the scheduler output:$l\n", $Session_id);
                if ((get_conf_with_default_param("SCHEDULER_LAUNCHER_OPTIMIZATION", "yes") eq "yes")
                    and
                    ($l =~ m/^SCHEDRUN JOB_ID=(\d+) MOLDABLE_JOB_ID=(\d+) RESOURCES=([\d,]+)$/m)
                ) {
                    my $job_id_to_schedule          = $1;
                    my $moldable_job_id_to_schedule = $2;
                    my @job_resources_to_schedule   = split(',', $3);
                    my $skip                        = "";
                    foreach my $r (@job_resources_to_schedule) {
                        if (defined($besteffort_resource_occupation{$r})) {
                            $skip = "a besteffort job needs to be killed first";
                            last;
                        } elsif (vec($Absent_resource_id_vec, $r, 1)) {
                            $skip = "some nodes need to be booted first";
                        }
                    }
                    if ($skip eq "") {
                        if (
                            not OAR::IO::is_inner_job_with_container_not_ready(
                                $base, $job_id_to_schedule)
                        ) {
                            oar_info(
                                $Module_name, "Early launch of the job\n",
                                $Session_id,  $job_id_to_schedule);
                            my $job = OAR::IO::get_job($base, $job_id_to_schedule);
                            prepare_job_to_be_launched($base, $job_id_to_schedule,
                                $moldable_job_id_to_schedule, $job->{submission_time},
                                \@job_resources_to_schedule);
                        } else {
                            oar_info(
                                $Module_name,
                                "Not able to early launch inner job with container not running yet\n",
                                $Session_id,
                                $job_id_to_schedule);
                        }
                    } else {
                        oar_info($Module_name,
                            "Not able to early launch $job_id_to_schedule: $skip\n",
                            $Session_id, $job_id_to_schedule);
                    }
                }
            }
            close(READSCHED);
            $sched_exit_code   = $? >> 8;
            $sched_signal_num  = $? & 127;
            $sched_dumped_core = $? & 128;
        }
        if ((!defined($sched_pid)) or ($sched_signal_num != 0) or ($sched_dumped_core != 0)) {
            oar_error(
                $Module_name,
                "Error: execution of $policy failed (pid=" .
                  (defined($sched_pid) ? "undef" : $sched_pid) .
                  " signal=$sched_signal_num core_dump=$sched_dumped_core). Disabling queue $name (see `oarnotify')\n",
                $Session_id);
            OAR::IO::stop_a_queue($base, $name);
        }
        if ($sched_exit_code == 1) {
            $exit_code = 0;
        } elsif ($sched_exit_code != 0) {
            oar_error(
                $Module_name,
                "Error: Scheduler $policy returned a bad value: $sched_exit_code, at time $initial_time{sec}. Disabling queue $name (see `oarnotify')\n",
                $Session_id);
            OAR::IO::stop_a_queue($base, $name);
        }
        treate_waiting_reservation_jobs($name);
        check_reservation_jobs($name);

        my @stats_queue_end = OAR::IO::get_stats($base);
        my $stats_queue_str = OAR::IO::format_stats(@stats_queue_start, @stats_queue_end);
        my $start_queue_count_str = OAR::IO::format_count_jobs_of_queue($name,%start_job_counts);
        oar_info($Module_name, "End of processing for Queue $name ($stats_queue_str ; $start_queue_count_str)\n", $Session_id);

    } else {
        oar_info($Module_name, "Queue $name: No job\n", $Session_id);
    }
}

if ($exit_code == 0) {
    if (check_jobs_to_kill() == 1) {

        # We must kill besteffort jobs
        OAR::Tools::notify_tcp_socket($Remote_host, $Remote_port, "ChState");
        $exit_code = 2;
    } elsif (check_jobs_to_launch() == 1) {
        $exit_code = 0;
    }
}

#Update visu gantt tables
OAR::IO::update_gantt_visualization($base);
OAR::IO::disconnect($base_ro);

# Manage dynamic node feature
my $timeout_cmd = 10;
my $flagHulot   = 0;
if (is_conf("SCHEDULER_TIMEOUT")) {
    $timeout_cmd = get_conf("SCHEDULER_TIMEOUT");
}
if (
    (   is_conf("SCHEDULER_NODE_MANAGER_SLEEP_CMD") or
        (get_conf("ENERGY_SAVING_INTERNAL") eq "yes" and
            is_conf("ENERGY_SAVING_NODE_MANAGER_SLEEP_CMD"))
    ) and
    is_conf("SCHEDULER_NODE_MANAGER_SLEEP_TIME") and
    is_conf("SCHEDULER_NODE_MANAGER_IDLE_TIME")
) {

    # Look at nodes that are unused for a duration
    my $idle_duration  = get_conf("SCHEDULER_NODE_MANAGER_IDLE_TIME");
    my $sleep_duration = get_conf("SCHEDULER_NODE_MANAGER_SLEEP_TIME");

    my $tmp_time = $current_time_sec - $idle_duration;
    my @node_halt;
    my @nodes        = OAR::IO::get_nodes_to_halt($base, $current_time_sec, $sleep_duration);
    my @wakeup_dates = OAR::IO::get_last_wake_up_date_of_nodes($base, \@nodes);

    foreach my $last_wakeup (@wakeup_dates) {

        # Search if node has not been woken up recently
        if ($last_wakeup->{'date'} < $tmp_time) {
            my $wakeup_date = $last_wakeup->{'date'};
            if (!defined($wakeup_date) or ($wakeup_date < $tmp_time)) {
                push(@node_halt, $last_wakeup->{'hostname'});
            }
        }
    }

    if ($#node_halt >= 0) {
        oar_info($Module_name, "Powering off some nodes (energy saving): @node_halt\n",
            $Session_id);

        # Using the built-in energy saving module to shut down nodes
        if (get_conf_with_default_param("ENERGY_SAVING_INTERNAL", "no") eq "yes") {
            if (OAR::Modules::Hulot::halt_nodes(\@node_halt)) {
                oar_error($Module_name,
                    "Error: communication problem with the energy saving module (Hulot)\n",
                    $Session_id);
            }
            $flagHulot = 1;
        }

        # Not using the built-in energy saving module to shut down nodes
        else {
            my $cmd = get_conf("SCHEDULER_NODE_MANAGER_SLEEP_CMD");
            if (!defined(OAR::Tools::fork_and_feed_stdin($cmd, $timeout_cmd, \@node_halt))) {
                oar_error(
                    $Module_name,
                    "Error: command $cmd timed out (${timeout_cmd}s) while trying to poweroff some nodes\n",
                    $Session_id);
            }
        }
    }
}

if (is_conf("SCHEDULER_NODE_MANAGER_WAKE_UP_CMD") or
    (get_conf("ENERGY_SAVING_INTERNAL") eq "yes" and
        is_conf("ENERGY_SAVING_NODE_MANAGER_WAKE_UP_CMD"))
) {
# Get nodes which the scheduler wants to schedule jobs to, but which are in the Absent state, to wake them up.
    my $wakeup_time = get_conf_with_default_param("SCHEDULER_NODE_MANAGER_WAKEUP_TIME", 1);
    my @nodes = OAR::IO::get_gantt_hostname_to_wake_up($base, $current_time_sec, $wakeup_time);

    if ($#nodes >= 0) {
        oar_info($Module_name, "Waking-up some nodes: @nodes\n", $Session_id);

        # Using the built-in energy saving module to wake up nodes
        if (get_conf_with_default_param("ENERGY_SAVING_INTERNAL", "no") eq "yes") {
            if (OAR::Modules::Hulot::wake_up_nodes(\@nodes)) {
                oar_error($Module_name,
                    "Error: Communication problem with the energy saving module (Hulot)\n",
                    $Session_id);
            }
            $flagHulot = 1;
        }

        # Not using the built-in energy saving module to wake up nodes
        else {
            my $cmd = get_conf("SCHEDULER_NODE_MANAGER_WAKE_UP_CMD");
            if (!defined(OAR::Tools::fork_and_feed_stdin($cmd, $timeout_cmd, \@nodes))) {
                oar_error(
                    $Module_name,
                    "Error: command $cmd timed out (${timeout_cmd}s) while trying to wake-up some nodes\n",
                    $Session_id);
            }
        }
    } else {
        oar_info($Module_name, "No node to wake-up\n", $Session_id);
    }
}

# Send CHECK signal to Hulot if needed
if (!$flagHulot and (get_conf_with_default_param("ENERGY_SAVING_INTERNAL", "no") eq "yes")) {
    if (OAR::Modules::Hulot::check()) {
        oar_error($Module_name,
            "Error: communication problem with the energy saving module (Hulot)\n", $Session_id);
    }
}

# Search jobs to resume
foreach my $j (OAR::IO::get_jobs_in_state($base, "Resuming")) {
    my @other_jobs = OAR::IO::get_jobs_on_resuming_job_resources($base, $j->{job_id});

    # TODO: look for timesharing other jobs. What do we do?????
    if ($#other_jobs < 0) {

        # We can resume the job
        oar_info($Module_name, "Resuming job\n", $Session_id, "$j->{job_id}");
        my $jobtypes = OAR::IO::get_job_types_hash($base, $j->{job_id});
        if (defined($jobtypes->{noop})) {
            OAR::IO::resume_job_action($base, $j->{job_id});
            oar_info($Module_name, "Resume NOOP job OK\n", $Session_id, "$j->{job_id}");
        } else {
            ###############
            # RESUME PART #
            ###############
            my $script  = get_conf("JUST_BEFORE_RESUME_EXEC_FILE");
            my $timeout = get_conf("SUSPEND_RESUME_SCRIPT_TIMEOUT");
            $timeout = OAR::Tools::get_default_suspend_resume_script_timeout()
              if (!defined($timeout));
            my $skip = 0;
            if (defined($script)) {

                # Launch admin script
                my $script_error = 0;
                eval {
                    $SIG{ALRM} = sub { die "alarm\n" };
                    alarm($timeout);
                    oar_info($Module_name, "Running post suspend script: `$script $j->{job_id}'\n",
                        $Session_id, "$j->{job_id}");
                    $script_error = system("$script script $j->{job_id}");
                    oar_info($Module_name, "`$script $j->{job_id}' terminated\n",
                        $Session_id, "$j->{job_id}");
                    alarm(0);
                };
                if ($@ || ($script_error != 0)) {
                    my $log_str = "Suspend script error: $@; return code = $script_error\n";
                    my $str     = "[MetaSched] [$j->{job_id}] $log_str";
                    oar_error($Module_name, $log_str, $Session_id, $j->{job_id});
                    OAR::IO::add_new_event($base, "RESUME_SCRIPT_ERROR", $j->{job_id}, $str);
                    OAR::IO::frag_job($base, $j->{job_id});
                    OAR::Tools::notify_tcp_socket($Remote_host, $Remote_port, "Qdel");
                    $skip = 1;
                }
            }

            if ((defined($Cpuset_field)) and ($skip == 0)) {
                my $cpuset_name = OAR::IO::get_job_cpuset_name($base, $j->{job_id})
                  if (defined($Cpuset_field));
                my $cpuset_nodes =
                  OAR::IO::get_cpuset_values_for_a_moldable_job($base, $Cpuset_field,
                    $j->{assigned_moldable_job});
                my $suspend_data_hash = {
                    name             => $cpuset_name,
                    job_id           => $j->{job_id},
                    job_user         => $j->{job_user},
                    oarexec_pid_file => OAR::Tools::get_oar_pid_file_name($j->{job_id}), };
                if (defined($cpuset_nodes)) {
                    my $taktuk_cmd   = get_conf("TAKTUK_CMD");
                    my $suspend_file = get_conf("SUSPEND_RESUME_FILE");
                    $suspend_file = OAR::Tools::get_default_suspend_resume_file()
                      if (!defined($suspend_file));
                    $suspend_file = "$ENV{OARDIR}/$suspend_file" if ($suspend_file !~ /^\//);
                    my ($tag, @bad) = OAR::IO::manage_remote_commands(
                        [ keys(%{$cpuset_nodes}) ], $suspend_data_hash,
                        $suspend_file,              "resume",
                        $Openssh_cmd,               $taktuk_cmd,
                        $base,                      $Module_name,
                        $Session_id);
                    if ($tag == 0) {
                        my $log_str = "Bad suspend/resume file: $suspend_file\n";
                        my $str     = "[MetaSched] [SUSPEND_RESUME] [$j->{job_id}] $log_str\n";
                        oar_error($Module_name, $log_str, $Session_id, $j->{job_id});
                        OAR::IO::add_new_event($base, "SUSPEND_RESUME_MANAGER_FILE", $j->{job_id},
                            $str);
                    } else {
                        if (($#bad < 0)) {
                            OAR::IO::resume_job_action($base, $j->{job_id});
                        } else {
                            my $log_str = "Error on several nodes: @bad\n";
                            my $str     = "[MetaSched] [SUSPEND_RESUME] [$j->{job_id}] $log_str\n";
                            oar_error($Module_name, $log_str, $Session_id, $j->{job_id});
                            OAR::IO::add_new_event_with_host($base, "RESUME_ERROR", $j->{job_id},
                                $str, \@bad);
                            OAR::IO::frag_job($base, $j->{job_id});

                            # A Leon must be run
                            $exit_code = 2;
                        }
                    }
                }
            }
            #####################
            # RESUME PART, END  #
            #####################
        }
    }
}

# Notify oarsub -I when they will be launched
foreach my $j (OAR::IO::get_gantt_waiting_interactive_prediction_date($base)) {
    my ($addr, $port) = split(/:/, $j->{info_type});
    my $new_start_prediction = OAR::IO::local_to_sql($j->{start_time});
    oar_info($Module_name,
        "Notifying user of the start prediction: $new_start_prediction (" . $j->{message} . ")\n",
        $Session_id, "$j->{job_id}");
    OAR::Tools::notify_tcp_socket($addr, $port,
        "[$initial_time{sql}] Start prediction: $new_start_prediction (" . $j->{message} . ")");
}

# Run the decisions
## Handle "toError" jobs
foreach my $j (OAR::IO::get_jobs_in_state($base, "toError")) {
    my ($addr, $port) = split(/:/, $j->{info_type});
    oar_info($Module_name, "Handle job $j->{job_id} toError state\n", $Session_id, $j->{job_id});
    if (($j->{job_type} eq "INTERACTIVE") or
        (($j->{job_type} eq "PASSIVE") and
            ($j->{reservation} eq "Scheduled"))
    ) {
        oar_info($Module_name,
            "Notify oarsub job (num:$j->{job_id}) in error; jobInfo=$j->{info_type}\n",
            $Session_id);
        if ((defined(OAR::Tools::notify_tcp_socket($addr, $port, "$j->{message}"))) or
            (defined(OAR::Tools::notify_tcp_socket($addr, $port, "BAD JOB")))) {
            oar_info($Module_name,
                "Cannot open connection to oarsub client for job $j->{job_id}!\n",
                $Session_id, $j->{job_id});
        }
    }
    oar_info($Module_name, "Set job $j->{job_id} to state Error\n", $Session_id, $j->{job_id});
    OAR::IO::set_job_state($base, $j->{job_id}, "Error");
}
## Treate toAckReservation jobs
foreach my $j (OAR::IO::get_jobs_in_state($base, "toAckReservation")) {
    my ($addr, $port) = split(/:/, $j->{info_type});
    oar_info($Module_name, "Handle job $j->{job_id} toAckReservation state\n",
        $Session_id, $j->{job_id});
    my $err = OAR::Tools::notify_tcp_socket($addr, $port, "GOOD RESERVATION");
    if (defined($err)) {
        oar_warn($Module_name,
            "Frag job $j->{job_id}, I cannot notify oarsub for the reservation ($err)\n",
            $Session_id);
        OAR::IO::add_new_event($base, "CANNOT_NOTIFY_OARSUB", $j->{job_id},
            "[MetaSched] Can not notify oarsub for the job $j->{job_id} ($err)");
        OAR::IO::lock_table($base, [ "frag_jobs", "event_logs", "jobs" ]);
        OAR::IO::frag_job($base, $j->{job_id});
        OAR::IO::unlock_table($base);
        $exit_code = 2;
    } else {
        oar_info(
            $Module_name,
            "Notify oarsub for a RESERVATION (idJob=$j->{job_id}) --> OK; jobInfo=$j->{info_type}\n",
            $Session_id);
        OAR::IO::set_job_state($base, $j->{job_id}, "Waiting");
        if ($j->{start_time} - 1 <= $current_time_sec) {
            $exit_code = 1 if ($exit_code == 0);
        }
    }
}
## Treate toLaunch jobs
foreach my $j (OAR::IO::get_jobs_in_state($base, "toLaunch")) {
    notify_to_run_job($base, $j->{job_id});
}

my @stats_end = OAR::IO::get_stats($base);
my $stats_end_str = OAR::IO::format_stats(@stats_start, @stats_end);
OAR::IO::disconnect($base);
oar_info($Module_name, "End of Meta Scheduler ($stats_end_str)\n", $Session_id);

exit($exit_code);
