resource: satisfiability for resources going up/down

Problem: When a resource goes down and a job requests
all of the resources, the job spec is currently
evaluated to be unsatisfiable. This is incorrect.
In MATCH_ALLOCATE_W_SATISFIABILITY mode, we check
a job's satisfiability by trying to schedule
the job as late as possible. However, when a
resource is down, it doesn't matter how late we
try to schedule this: this job just cannot be
scheduled and our system will deem it unsatisfiable.
Similarly in MATCH_ALLOCATE_ORELSE_RESERVE mode, we
deem a job unsatisfiable when we iterate through
all of the scheduled points but find no scheduled
point that satisfies the request. Again, this
will not work when there is a resource that
is marked "down".

Solution:
Expand the allocation type within jobmeta_t beyond
boolean (allocation vs. reservation). If the allocation
type is satisfiability (AT_SATISFIABILITY), we just
ignore the current status of visiting resources
for matching so that those down resources will still
be considered for satisfiability checks.

qmanager/policies/queue_policy_bf_base_impl.hpp

@@ -90,10 +90,16 @@ queue_policy_bf_base_t<reapi_type>::allocate_orelse_reserve (void *h,
             iter = to_running (iter, use_alloced_queue);
         }
     } else {
-        // The request must be rejected. The job is enqueued into
-        // rejected job queue to the upper layer to react on this.
-        iter = to_rejected (iter, (errno == ENODEV)? "unsatisfiable"
-                                                   : "match error");
+        if (errno != EBUSY) {
+            // The request must be rejected. The job is enqueued into
+            // rejected job queue to the upper layer to react on this.
+            iter = to_rejected (iter, (errno == ENODEV)? "unsatisfiable"
+                                                       : "match error");
+        } else {
+            // This can happen if there are "down" resources.
+            // The semantics of our backfill policies is to skip this job
+            iter++;
+        }
     }
     return iter;
 }

resource/traversers/dfu.cpp

@@ -65,7 +65,7 @@ int dfu_traverser_t::schedule (Jobspec::Jobspec &jobspec,
     case match_op_t::MATCH_ALLOCATE_W_SATISFIABILITY: {
         /* With satisfiability check */
         errno = EBUSY;
-        meta.allocate = false;
+        meta.alloc_type = jobmeta_t::alloc_type_t::AT_SATISFIABILITY;
         p = (*get_graph ())[root].idata.subplans.at (dom);
         meta.at = planner_multi_base_time (p)
                   + planner_multi_duration (p) - meta.duration - 1;
@@ -79,7 +79,7 @@ int dfu_traverser_t::schedule (Jobspec::Jobspec &jobspec,
     case match_op_t::MATCH_ALLOCATE_ORELSE_RESERVE: {
         /* Or else reserve */
         errno = 0;
-        meta.allocate = false;
+        meta.alloc_type = jobmeta_t::alloc_type_t::AT_ALLOC_ORELSE_RESERVE;
         t = meta.at + 1;
         p = (*get_graph ())[root].idata.subplans.at (dom);
         len = planner_multi_resources_len (p);
@@ -91,8 +91,16 @@ int dfu_traverser_t::schedule (Jobspec::Jobspec &jobspec,
             rc = detail::dfu_impl_t::select (jobspec, root, meta, x);
         }
         // The planner layer returns ENOENT when no scheduleable point exists
-        // Turn this into ENODEV
-        errno = (rc < 0 && errno == ENOENT)? ENODEV : errno;
+        if (rc < 0 && errno == ENOENT) {
+            errno = EBUSY;
+            meta.alloc_type = jobmeta_t::alloc_type_t::AT_SATISFIABILITY;
+            meta.at = planner_multi_base_time (p)
+                      + planner_multi_duration (p) - duration - 1;
+            if (detail::dfu_impl_t::select (jobspec, root, meta, x) < 0) {
+                errno = (errno == EBUSY)? ENODEV : errno;
+                detail::dfu_impl_t::update ();
+            }
+        }
         break;
     }
     case match_op_t::MATCH_ALLOCATE:
@@ -240,7 +248,7 @@ int dfu_traverser_t::run (Jobspec::Jobspec &jobspec,
     bool x = detail::dfu_impl_t::exclusivity (jobspec.resources, root);
     std::unordered_map<std::string, int64_t> dfv;
     detail::dfu_impl_t::prime_jobspec (jobspec.resources, dfv);
-    meta.build (jobspec, true, jobid, *at);
+    meta.build (jobspec, detail::jobmeta_t::alloc_type_t::AT_ALLOC, jobid, *at);
     if ( (rc = schedule (jobspec, meta, x, op, root, dfv)) ==  0) {
         *at = meta.at;
         rc = detail::dfu_impl_t::update (root, writers, meta);

resource/traversers/dfu_impl.cpp

@@ -202,7 +202,8 @@ int dfu_impl_t::prune (const jobmeta_t &meta, bool exclusive,
     int rc = 0;
     // Prune by the visiting resource vertex's availability
     // If resource is not UP, no reason to descend further.
-    if ((*m_graph)[u].status != resource_pool_t::status_t::UP) {
+    if (meta.alloc_type != jobmeta_t::alloc_type_t::AT_SATISFIABILITY
+        && (*m_graph)[u].status != resource_pool_t::status_t::UP) {
         rc = -1;
         goto done;
     }

resource/traversers/dfu_impl.hpp

@@ -53,7 +53,14 @@ enum class match_kind_t { RESOURCE_MATCH,
                           PRESTINE_NONE_MATCH };
 
 struct jobmeta_t {
-    bool allocate = true;
+
+    enum class alloc_type_t : int {
+        AT_ALLOC = 0,
+        AT_ALLOC_ORELSE_RESERVE = 1,
+        AT_SATISFIABILITY = 2
+    };
+
+    alloc_type_t alloc_type = alloc_type_t::AT_ALLOC;
     int64_t jobid = -1;
     int64_t at = -1;
     uint64_t duration = SYSTEM_DEFAULT_DURATION; // will need config ultimately
@@ -66,11 +73,12 @@ struct jobmeta_t {
         return m_queue;
     }
 
-    void build (Jobspec::Jobspec &jobspec, bool alloc, int64_t id, int64_t t)
+    void build (Jobspec::Jobspec &jobspec,
+                alloc_type_t alloc, int64_t id, int64_t t)
     {
         at = t;
         jobid = id;
-        allocate = alloc;
+        alloc_type = alloc;
         if (jobspec.attributes.system.duration == 0.0f
             || jobspec.attributes.system.duration > (double)UINT64_MAX)
             duration = SYSTEM_MAX_DURATION; // need config support ultimately

resource/traversers/dfu_impl_update.cpp

@@ -123,6 +123,8 @@ int dfu_impl_t::upd_plan (vtx_t u, const subsystem_t &s, unsigned int needs,
                           bool excl,  const jobmeta_t &jobmeta, bool full,
                           int &n)
 {
+    int rc = 0;
+
     if (excl) {
 
         n++;
@@ -146,14 +148,28 @@ int dfu_impl_t::upd_plan (vtx_t u, const subsystem_t &s, unsigned int needs,
                 m_err_msg += strerror (errno);
                 m_err_msg += "\n";
             }
-            return -1;
+            rc = -1;
+            goto done;
         }
-        if (jobmeta.allocate)
+
+        switch (jobmeta.alloc_type) {
+        case jobmeta_t::alloc_type_t::AT_ALLOC:
             (*m_graph)[u].schedule.allocations[jobmeta.jobid] = span;
-        else
+            break;
+        case jobmeta_t::alloc_type_t::AT_ALLOC_ORELSE_RESERVE:
             (*m_graph)[u].schedule.reservations[jobmeta.jobid] = span;
+            break;
+        case jobmeta_t::alloc_type_t::AT_SATISFIABILITY:
+            break;
+        default:
+            rc = -1;
+            errno = EINVAL;
+            break;
+        }
     }
-    return 0;
+
+done:
+    return rc;
 }
 
 int dfu_impl_t::accum_to_parent (vtx_t u, const subsystem_t &subsystem,
@@ -512,13 +528,15 @@ int dfu_impl_t::update (vtx_t root, std::shared_ptr<match_writers_t> &writers,
     unsigned int needs = 0;
     std::map<std::string, int64_t> dfu;
     const std::string &dom = m_match->dom_subsystem ();
+    bool rsv = (jobmeta.alloc_type
+                 == jobmeta_t::alloc_type_t::AT_ALLOC_ORELSE_RESERVE);
 
     tick ();
     if ( (rc = reader->update (m_graph_db->resource_graph,
                                m_graph_db->metadata, str,
                                jobmeta.jobid, jobmeta.at,
                                jobmeta.duration,
-                               !jobmeta.allocate, m_best_k_cnt)) != 0) {
+                               rsv, m_best_k_cnt)) != 0) {
         m_err_msg += reader->err_message ();
         reader->clear_err_message ();
         return rc;