mon: automatically commit used pending_keys

Signed-off-by: Sage Weil <[email protected]>

src/messages/MMonUsedPendingKeys.h

@@ -0,0 +1,48 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <[email protected]>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#pragma once
+
+#include "messages/PaxosServiceMessage.h"
+
+class MMonUsedPendingKeys final : public PaxosServiceMessage {
+public:
+  std::map<EntityName,CryptoKey> used_pending_keys;
+
+  MMonUsedPendingKeys() : PaxosServiceMessage{MSG_MON_USED_PENDING_KEYS, 0}
+  {}
+private:
+  ~MMonUsedPendingKeys() final {}
+
+public:
+  std::string_view get_type_name() const override { return "used_pending_keys"; }
+  void print(std::ostream& out) const override {
+    out << "used_pending_keys(" << used_pending_keys.size() << " keys)";
+  }
+
+  void decode_payload() override {
+    using ceph::decode;
+    auto p = payload.cbegin();
+    paxos_decode(p);
+    decode(used_pending_keys, p);
+  }
+  void encode_payload(uint64_t features) override {
+    using ceph::encode;
+    paxos_encode();
+    encode(used_pending_keys, payload);
+  }
+private:
+  template<class T, typename... Args>
+  friend boost::intrusive_ptr<T> ceph::make_message(Args&&... args);
+};

src/mon/AuthMonitor.cc

@@ -25,6 +25,7 @@
 #include "messages/MAuth.h"
 #include "messages/MAuthReply.h"
 #include "messages/MMonGlobalID.h"
+#include "messages/MMonUsedPendingKeys.h"
 #include "msg/Messenger.h"
 
 #include "auth/AuthServiceHandler.h"
@@ -91,6 +92,38 @@ bool AuthMonitor::check_rotate()
   return false;
 }
 
+void AuthMonitor::process_used_pending_keys(
+  const std::map<EntityName,CryptoKey>& used_pending_keys)
+{
+  for (auto& [name, used_key] : used_pending_keys) {
+    dout(10) << __func__ << " used pending_key for " << name << dendl;
+    KeyServerData::Incremental inc;
+    inc.op = KeyServerData::AUTH_INC_ADD;
+    inc.name = name;
+
+    mon.key_server.get_auth(name, inc.auth);
+    for (auto& p : pending_auth) {
+      if (p.inc_type == AUTH_DATA) {
+	KeyServerData::Incremental auth_inc;
+	auto q = p.auth_data.cbegin();
+	decode(auth_inc, q);
+	if (auth_inc.op == KeyServerData::AUTH_INC_ADD &&
+	    auth_inc.name == name) {
+	  dout(10) << __func__ << "  starting with pending uncommitted" << dendl;
+	  inc.auth = auth_inc.auth;
+	}
+      }
+    }
+    if (stringify(inc.auth.pending_key) == stringify(used_key)) {
+      dout(10) << __func__ << " committing pending_key -> key for "
+	       << name << dendl;
+      inc.auth.key = inc.auth.pending_key;
+      inc.auth.pending_key.clear();
+      push_cephx_inc(inc);
+    }
+  }
+}
+
 /*
  Tick function to update the map based on performance every N seconds
 */
@@ -114,10 +147,26 @@ void AuthMonitor::tick()
       propose = true;
     } else {
       dout(10) << __func__ << "requesting more ids from leader" << dendl;
-      int leader = mon.get_leader();
       MMonGlobalID *req = new MMonGlobalID();
       req->old_max_id = max_global_id;
-      mon.send_mon_message(req, leader);
+      mon.send_mon_message(req, mon.get_leader());
+    }
+  }
+
+  if (mon.monmap->min_mon_release >= ceph_release_t::quincy) {
+    std::map<EntityName,CryptoKey> used_pending_keys;
+    mon.key_server.get_used_pending_keys(&used_pending_keys);
+    if (!used_pending_keys.empty()) {
+      dout(10) << __func__ << " " << used_pending_keys.size() << " used pending_keys"
+	       << dendl;
+      if (mon.is_leader()) {
+	process_used_pending_keys(used_pending_keys);
+	propose = true;
+      } else {
+	MMonUsedPendingKeys *req = new MMonUsedPendingKeys();
+	req->used_pending_keys = used_pending_keys;
+	mon.send_mon_message(req, mon.get_leader());
+      }
     }
   }
 
@@ -142,6 +191,7 @@ void AuthMonitor::on_active()
     return;
 
   mon.key_server.start_server();
+  mon.key_server.clear_used_pending_keys();
 
   if (is_writeable()) {
     bool propose = false;
@@ -523,6 +573,9 @@ bool AuthMonitor::preprocess_query(MonOpRequestRef op)
   case MSG_MON_GLOBAL_ID:
     return false;
 
+  case MSG_MON_USED_PENDING_KEYS:
+    return false;
+
   default:
     ceph_abort();
     return true;
@@ -544,6 +597,8 @@ bool AuthMonitor::prepare_update(MonOpRequestRef op)
     }
   case MSG_MON_GLOBAL_ID:
     return prepare_global_id(op);
+  case MSG_MON_USED_PENDING_KEYS:
+    return prepare_used_pending_keys(op);
   case CEPH_MSG_AUTH:
     return prep_auth(op, true);
   default:
@@ -1892,7 +1947,14 @@ bool AuthMonitor::prepare_global_id(MonOpRequestRef op)
 {
   dout(10) << "AuthMonitor::prepare_global_id" << dendl;
   increase_max_global_id();
+  return true;
+}
 
+bool AuthMonitor::prepare_used_pending_keys(MonOpRequestRef op)
+{
+  dout(10) << __func__ << " " << op << dendl;
+  auto m = op->get_req<MMonUsedPendingKeys>();
+  process_used_pending_keys(m->used_pending_keys);
   return true;
 }
 

src/mon/AuthMonitor.h

@@ -151,6 +151,8 @@ class AuthMonitor : public PaxosService {
   void _set_mon_num_rank(int num, int rank); ///< called under mon->auth_lock
 
 private:
+  bool prepare_used_pending_keys(MonOpRequestRef op);
+
   // propose pending update to peers
   void encode_pending(MonitorDBStore::TransactionRef t) override;
   void encode_full(MonitorDBStore::TransactionRef t) override;
@@ -165,6 +167,7 @@ class AuthMonitor : public PaxosService {
   bool prepare_command(MonOpRequestRef op);
 
   bool check_rotate();
+  void process_used_pending_keys(const std::map<EntityName,CryptoKey>& keys);
 
   bool entity_is_pending(EntityName& entity);
   int exists_and_matches_entity(

src/mon/Monitor.cc

@@ -4510,6 +4510,7 @@ void Monitor::dispatch_op(MonOpRequestRef op)
   switch (op->get_req()->get_type()) {
     // auth
     case MSG_MON_GLOBAL_ID:
+    case MSG_MON_USED_PENDING_KEYS:
     case CEPH_MSG_AUTH:
       op->set_type_service();
       /* no need to check caps here */

src/msg/Message.cc

@@ -115,6 +115,7 @@
 #include "messages/MMonSubscribe.h"
 #include "messages/MMonSubscribeAck.h"
 #include "messages/MMonGlobalID.h"
+#include "messages/MMonUsedPendingKeys.h"
 #include "messages/MClientSession.h"
 #include "messages/MClientReconnect.h"
 #include "messages/MClientRequest.h"
@@ -644,6 +645,9 @@ Message *decode_message(CephContext *cct,
   case MSG_MON_GLOBAL_ID:
     m = make_message<MMonGlobalID>();
     break; 
+  case MSG_MON_USED_PENDING_KEYS:
+    m = make_message<MMonUsedPendingKeys>();
+    break; 
 
     // clients
   case CEPH_MSG_MON_SUBSCRIBE:

src/msg/Message.h

@@ -58,6 +58,7 @@
 #define MSG_GETPOOLSTATSREPLY      59
 
 #define MSG_MON_GLOBAL_ID          60
+#define MSG_MON_USED_PENDING_KEYS  141
 
 #define MSG_ROUTE                  47
 #define MSG_FORWARD                46