rgw/lua: allow for multiple RGWs to install packages in the same host

each RGW will install the luarocks packages in its own directory

Signed-off-by: Yuval Lifshitz <[email protected]>

doc/radosgw/lua-scripting.rst

@@ -15,7 +15,7 @@ By default, all lua standard libraries are available in the script, however, in
   - All packages in the allowlist are being re-installed using the luarocks package manager on radosgw restart. Therefore a restart is needed for adding or removing of packages to take effect 
   - To add a package that contains C source code that needs to be compiled, use the `--allow-compilation` flag. In this case a C compiler needs to be available on the host
   - Lua packages are installed in, and used from, a directory local to the radosgw. Meaning that lua packages in the allowlist are separated from any lua packages available on the host.
-    By default, this directory would be `/tmp/luarocks/`, and could be set to a different location via the `rgw_luarocks_location` configuration parameter. 
+    By default, this directory would be `/tmp/luarocks/<entity name>`. Its prefix part (`/tmp/luarocks/`) could be set to a different location via the `rgw_luarocks_location` configuration parameter. 
     Note that this parameter should not be set to one of the default locations where luarocks install packages (e.g. `$HOME/.luarocks`, `/usr/lib64/lua`, `/usr/share/lua`)
 
 

src/rgw/rgw_admin.cc

@@ -9374,7 +9374,7 @@ int main(int argc, const char **argv)
       return -rc;
     }
 #else
-    cerr << "ERROR: adding lua packages in not permitted" << std::endl;
+    cerr << "ERROR: adding lua packages is not permitted" << std::endl;
     return EPERM;
 #endif
   }

src/rgw/rgw_lua.cc

@@ -165,7 +165,6 @@ int add_package(rgw::sal::RGWRadosStore* store, optional_yield y, const std::str
 
   std::string line;
   bool package_found = false;
-  // TODO: yield on reading the output
   while (c.running() && std::getline(is, line) && !line.empty()) {
     package_found = true;
   }
@@ -230,9 +229,9 @@ int list_packages(rgw::sal::RGWRadosStore* store, optional_yield y, packages_t&
 
 int install_packages(rgw::sal::RGWRadosStore* store, optional_yield y, packages_t& failed_packages, std::string& output) {
   // luarocks directory cleanup
-  const auto& luarocks_location = g_conf().get_val<std::string>("rgw_luarocks_location");
   boost::system::error_code ec;
-  boost::filesystem::remove_all(luarocks_location, ec);
+  const auto& luarocks_path = store->get_luarocks_path();
+  boost::filesystem::remove_all(luarocks_path, ec);
   if (ec.value() != 0 && ec.value() != ENOENT) {
     output.append("failed to clear luarock directory: ");
     output.append(ec.message());
@@ -258,13 +257,13 @@ int install_packages(rgw::sal::RGWRadosStore* store, optional_yield y, packages_
   // the lua rocks install dir will be created by luarocks the first time it is called
   for (const auto& package : packages) {
     bp::ipstream is;
-    bp::child c(p, "install", "--lua-version", CEPH_LUA_VERSION, "--tree", luarocks_location, "--deps-mode", "one", package, 
+    bp::child c(p, "install", "--lua-version", CEPH_LUA_VERSION, "--tree", luarocks_path, "--deps-mode", "one", package, 
         bp::std_in.close(),
         (bp::std_err & bp::std_out) > is);
 
-    // TODO: yield until wait returns
+    // once package reload is supported, code should yield when reading output
     std::string line = "CMD: luarocks install --lua-version " + std::string(CEPH_LUA_VERSION) + std::string(" --tree ") + 
-      luarocks_location + " --deps-mode one " + package;
+      luarocks_path + " --deps-mode one " + package;
 
     do {
       if (!line.empty()) {

src/rgw/rgw_lua_request.cc

@@ -774,15 +774,16 @@ int execute(
     OpsLogSocket* olog,
     req_state* s, 
     const char* op_name,
-    const std::string& script,
-    const std::string& package_path)
+    const std::string& script)
 
 {
   auto L = luaL_newstate();
   lua_state_guard lguard(L);
 
   open_standard_libs(L);
-  set_package_path(L, package_path);
+  set_package_path(L, store ?
+      store->get_luarocks_path() : 
+      "");
 
   create_debug_action(L, s->cct);  
 

src/rgw/rgw_lua_request.h

@@ -19,8 +19,7 @@ int execute(
     OpsLogSocket* olog,
     req_state *s, 
     const char* op_name,
-    const std::string& script,
-    const std::string& package_path);
+    const std::string& script);
 
 }
 

src/rgw/rgw_main.cc

@@ -407,6 +407,12 @@ int radosgw_Main(int argc, const char **argv)
 #endif
   }
 
+  const auto& luarocks_path = g_conf().get_val<std::string>("rgw_luarocks_location");
+  if (luarocks_path.empty()) {
+    store->set_luarocks_path("");
+  } else {
+    store->set_luarocks_path(luarocks_path+"/"+g_conf()->name.to_str());
+  }
 #ifdef WITH_RADOSGW_LUA_PACKAGES
   rgw::lua::packages_t failed_packages;
   std::string output;

src/rgw/rgw_process.cc

@@ -225,7 +225,6 @@ int process_request(rgw::sal::RGWRadosStore* const store,
                                                frontend_prefix,
                                                client_io, &mgr, &init_error);
   rgw::dmclock::SchedulerCompleter c;
-  const auto& lua_package_path = g_conf().get_val<std::string>("rgw_luarocks_location"); 
 
   if (init_error != 0) {
     abort_early(s, nullptr, init_error, nullptr, yield);
@@ -249,7 +248,7 @@ int process_request(rgw::sal::RGWRadosStore* const store,
     } else if (rc < 0) {
       ldpp_dout(op, 5) << "WARNING: failed to read pre request script. error: " << rc << dendl;
     } else {
-      rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script, lua_package_path);
+      rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script);
       if (rc < 0) {
         ldpp_dout(op, 5) << "WARNING: failed to execute pre request script. error: " << rc << dendl;
       }
@@ -317,7 +316,7 @@ int process_request(rgw::sal::RGWRadosStore* const store,
     } else if (rc < 0) {
       ldpp_dout(op, 5) << "WARNING: failed to read post request script. error: " << rc << dendl;
     } else {
-      rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script, lua_package_path);
+      rc = rgw::lua::request::execute(store, rest, olog, s, op->name(), script);
       if (rc < 0) {
         ldpp_dout(op, 5) << "WARNING: failed to execute post request script. error: " << rc << dendl;
       }

src/rgw/rgw_sal.h

@@ -110,6 +110,11 @@ class RGWStore : public DoutPrefixProvider {
     virtual void finalize(void)=0;
 
     virtual CephContext *ctx(void)=0;
+
+    // get the location of where lua packages are installed
+    virtual const std::string& get_luarocks_path() const = 0;
+    // set the location of where lua packages are installed
+    virtual void set_luarocks_path(const std::string& path) = 0;
 };
 
 class RGWUser {

src/rgw/rgw_sal_rados.h

@@ -263,6 +263,7 @@ class RGWRadosStore : public RGWStore {
   private:
     RGWRados *rados;
     RGWUserCtl *user_ctl;
+    std::string luarocks_path;
 
   public:
     RGWRadosStore()
@@ -331,6 +332,13 @@ class RGWRadosStore : public RGWStore {
     CephContext* get_cct() const override { return rados->ctx(); }
     unsigned get_subsys() const override { return ceph_subsys_rgw; }
 
+    const std::string& get_luarocks_path() const override {
+      return luarocks_path;
+    }
+
+    void set_luarocks_path(const std::string& path) override {
+      luarocks_path = path;
+    }
 };
 
 class MPRadosSerializer : public MPSerializer {

src/test/rgw/test_rgw_lua.cc

@@ -80,7 +80,7 @@ TEST(TestRGWLua, EmptyScript)
   uint64_t id = 0;
   req_state s(cct, &e, id); 
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -95,7 +95,7 @@ TEST(TestRGWLua, SyntaxError)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -107,7 +107,7 @@ TEST(TestRGWLua, Hello)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -119,7 +119,7 @@ TEST(TestRGWLua, RGWDebugLogNumber)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -131,7 +131,7 @@ TEST(TestRGWLua, RGWDebugNil)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -145,7 +145,7 @@ TEST(TestRGWLua, URI)
   DEFINE_REQ_STATE;
   s.decoded_uri = "http://hello.world/";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -164,7 +164,7 @@ TEST(TestRGWLua, Response)
   s.err.err_code = "Bad Request";
   s.err.message = "This is a bad request";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -179,7 +179,7 @@ TEST(TestRGWLua, SetResponse)
   DEFINE_REQ_STATE;
   s.err.message = "this is a bad request";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -193,7 +193,7 @@ TEST(TestRGWLua, SetRGWId)
   DEFINE_REQ_STATE;
   s.host_id = "foo";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_NE(rc, 0);
 }
 
@@ -206,7 +206,7 @@ TEST(TestRGWLua, InvalidField)
   DEFINE_REQ_STATE;
   s.host_id = "foo";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -218,7 +218,7 @@ TEST(TestRGWLua, InvalidSubField)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "kaboom", script);
   ASSERT_EQ(rc, -1);
 }
 
@@ -252,7 +252,7 @@ TEST(TestRGWLua, Bucket)
   b.bucket_id = "myid"; 
   s.bucket.reset(new sal::RGWRadosBucket(nullptr, b));
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -275,7 +275,7 @@ TEST(TestRGWLua, GenericAttributes)
   s.generic_attrs["goodbye"] = "cruel world";
   s.generic_attrs["ka"] = "boom";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -298,7 +298,7 @@ TEST(TestRGWLua, Environment)
   s.env["goodbye"] = "cruel world";
   s.env["ka"] = "boom";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -319,7 +319,7 @@ TEST(TestRGWLua, Tags)
   s.tagset.add_tag("goodbye", "cruel world");
   s.tagset.add_tag("ka", "boom");
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -332,7 +332,7 @@ TEST(TestRGWLua, TagsNotWriteable)
   DEFINE_REQ_STATE;
   s.tagset.add_tag("hello", "world");
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_NE(rc, 0);
 }
 
@@ -358,7 +358,7 @@ TEST(TestRGWLua, Metadata)
   s.info.x_meta_map["foo"] = "bar";
   s.info.x_meta_map["ka"] = "boom";
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -409,7 +409,7 @@ TEST(TestRGWLua, Acl)
   s.user_acl->get_acl().add_grant(&grant3);
   s.user_acl->get_acl().add_grant(&grant4);
   s.user_acl->get_acl().add_grant(&grant5);
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -453,7 +453,7 @@ TEST(TestRGWLua, UseFunction)
   s.object_acl->get_owner().set_name("user five");
   s.object_acl->get_owner().set_id(rgw_user("tenant5", "user5"));
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -474,7 +474,7 @@ TEST(TestRGWLua, WithLib)
   b.name = "my-bucket-name-is-fish";
   s.bucket.reset(new sal::RGWRadosBucket(nullptr, b));
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_EQ(rc, 0);
 }
 
@@ -487,7 +487,7 @@ TEST(TestRGWLua, NotAllowedInLib)
 
   DEFINE_REQ_STATE;
 
-  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script, "");
+  const auto rc = lua::request::execute(nullptr, nullptr, nullptr, &s, "put_obj", script);
   ASSERT_NE(rc, 0);
 }
 #include <sys/socket.h>
@@ -563,11 +563,11 @@ TEST(TestRGWLua, OpsLog)
   s.cio = &ac; 
 	s.cct->_conf->rgw_ops_log_rados	= false;
 
-  auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script, "");
+  auto rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script);
   EXPECT_EQ(rc, 0);
 
 	s.err.http_ret = 400;
-  rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script, "");
+  rc = lua::request::execute(store.get(), nullptr, olog.get(), &s, "put_obj", script);
   EXPECT_EQ(rc, 0);
 
 	// give the socket client time to read