feat(cp): add dp cert details (Kong#11921)

* feat(cp): add dp cert details support for exposing dataplane certificate expiry date to `/clustering/data-planes` endpoint Fix: [FTI-5530](https://konghq.atlassian.net/browse/FTI-5530) Signed-off-by: tzssangglass <[email protected]>
phial3 · Nov 16, 2023 · a382576 · a382576
1 parent a7e7cb4
commit a382576
Show file tree

Hide file tree

Showing 13 changed files with 224 additions and 6 deletions.
diff --git a/changelog/unreleased/kong/cp-expose-dp-cert-details.yml b/changelog/unreleased/kong/cp-expose-dp-cert-details.yml
@@ -0,0 +1,5 @@
+message: |
+    **Clustering**: Expose data plane certificate expiry date on the control plane API.
+type: feature
+scope: Clustering
+
diff --git a/kong-3.6.0-0.rockspec b/kong-3.6.0-0.rockspec
@@ -278,6 +278,7 @@ build = {
     ["kong.db.migrations.core.019_320_to_330"] = "kong/db/migrations/core/019_320_to_330.lua",
     ["kong.db.migrations.core.020_330_to_340"] = "kong/db/migrations/core/020_330_to_340.lua",
     ["kong.db.migrations.core.021_340_to_350"] = "kong/db/migrations/core/021_340_to_350.lua",
+    ["kong.db.migrations.core.022_350_to_360"] = "kong/db/migrations/core/022_350_to_360.lua",
     ["kong.db.migrations.operations.200_to_210"] = "kong/db/migrations/operations/200_to_210.lua",
     ["kong.db.migrations.operations.212_to_213"] = "kong/db/migrations/operations/212_to_213.lua",
     ["kong.db.migrations.operations.280_to_300"] = "kong/db/migrations/operations/280_to_300.lua",

diff --git a/kong/clustering/control_plane.lua b/kong/clustering/control_plane.lua
@@ -77,6 +77,17 @@ local function is_timeout(err)
 end
 
 
+local function extract_dp_cert(cert)
+  local expiry_timestamp = cert:get_not_after()
+  -- values in cert_details must be strings
+  local cert_details = {
+    expiry_timestamp = expiry_timestamp,
+  }
+
+  return cert_details
+end
+
+
 function _M.new(clustering)
   assert(type(clustering) == "table",
          "kong.clustering is not instantiated")
@@ -183,7 +194,7 @@ _M.check_version_compatibility = compat.check_version_compatibility
 _M.check_configuration_compatibility = compat.check_configuration_compatibility
 
 
-function _M:handle_cp_websocket()
+function _M:handle_cp_websocket(cert)
   local dp_id = ngx_var.arg_node_id
   local dp_hostname = ngx_var.arg_node_hostname
   local dp_ip = ngx_var.remote_addr
@@ -230,6 +241,7 @@ function _M:handle_cp_websocket()
     return ngx_exit(ngx_CLOSE)
   end
 
+  local dp_cert_details = extract_dp_cert(cert)
   local dp_plugins_map = plugins_list_to_map(data.plugins)
   local config_hash = DECLARATIVE_EMPTY_CONFIG_HASH -- initial hash
   local last_seen = ngx_time()
@@ -247,6 +259,7 @@ function _M:handle_cp_websocket()
       version = dp_version,
       sync_status = sync_status, -- TODO: import may have been failed though
       labels = data.labels,
+      cert_details = dp_cert_details,
     }, { ttl = purge_delay })
     if not ok then
       ngx_log(ngx_ERR, _log_prefix, "unable to update clustering data plane status: ", err, log_suffix)

diff --git a/kong/clustering/init.lua b/kong/clustering/init.lua
@@ -63,13 +63,13 @@ end
 
 
 function _M:handle_cp_websocket()
-  local ok, err = self:validate_client_cert()
-  if not ok then
+  local cert, err = self:validate_client_cert()
+  if not cert then
     ngx_log(ngx_ERR, _log_prefix, err)
     return ngx_exit(444)
   end
 
-  return self.instance:handle_cp_websocket()
+  return self.instance:handle_cp_websocket(cert)
 end
 
 

diff --git a/kong/clustering/tls.lua b/kong/clustering/tls.lua
@@ -13,6 +13,8 @@ local constants = require("kong.constants")
 
 local ngx_log = ngx.log
 local WARN = ngx.WARN
+local tostring = tostring
+
 
 local OCSP_TIMEOUT = constants.CLUSTERING_OCSP_TIMEOUT
 
@@ -226,7 +228,7 @@ function tls.validate_client_cert(kong_config, cp_cert, dp_cert_pem)
     return nil, err
   end
 
-  return true
+  return cert, nil
 end
 
 

diff --git a/kong/db/migrations/core/022_350_to_360.lua b/kong/db/migrations/core/022_350_to_360.lua
@@ -0,0 +1,13 @@
+return {
+  postgres = {
+    up = [[
+      DO $$
+      BEGIN
+      ALTER TABLE IF EXISTS ONLY "clustering_data_planes" ADD "cert_details" JSONB;
+      EXCEPTION WHEN DUPLICATE_COLUMN THEN
+        -- Do nothing, accept existing state
+      END;
+      $$;
+    ]]
+  }
+}
diff --git a/kong/db/migrations/core/init.lua b/kong/db/migrations/core/init.lua
@@ -19,4 +19,5 @@ return {
   "019_320_to_330",
   "020_330_to_340",
   "021_340_to_350",
+  "022_350_to_360",
 }
diff --git a/kong/db/schema/entities/clustering_data_planes.lua b/kong/db/schema/entities/clustering_data_planes.lua
@@ -38,5 +38,13 @@ return {
                  description = "Custom key value pairs as meta-data for DPs.",
                },
     },
+    { cert_details = {
+        type = "record",
+        fields = {
+          { expiry_timestamp = { type = "number", timestamp = true, required = false } }
+        },
+        description = "Certificate details of the DPs.",
+      },
+    },
   },
 }
diff --git a/spec/01-unit/01-db/01-schema/13-cluster_status_spec.lua b/spec/01-unit/01-db/01-schema/13-cluster_status_spec.lua
@@ -66,4 +66,16 @@ describe("plugins", function()
     assert.is_true(ok)
     assert.is_nil(err)
   end)
+
+  it("accepts cert details", function()
+    local ok, err = validate({
+      ip = "127.0.0.1",
+      hostname = "dp.example.com",
+      cert_details = {
+        expiry_timestamp = 1897136778,
+      }
+    })
+    assert.is_true(ok)
+    assert.is_nil(err)
+  end)
 end)
diff --git a/spec/01-unit/19-hybrid/02-clustering_spec.lua b/spec/01-unit/19-hybrid/02-clustering_spec.lua
@@ -1,7 +1,6 @@
 local calculate_config_hash = require("kong.clustering.config_helper").calculate_config_hash
 local version = require("kong.clustering.compat.version")
 
-
 describe("kong.clustering.compat.version", function()
   it("correctly parses 3 or 4 digit version numbers", function()
     assert.equal(3000000000, version.string_to_number("3.0.0"))

diff --git a/spec/02-integration/03-db/13-cluster_status_spec.lua b/spec/02-integration/03-db/13-cluster_status_spec.lua
@@ -71,5 +71,46 @@ for _, strategy in helpers.each_strategy() do
         assert.is_nil(err)
       end)
     end)
+
+    describe("cert_details", function()
+      it(":upsert()", function()
+        local p, err =
+          db.clustering_data_planes:upsert(
+            {
+              id = "eb51145a-aaaa-bbbb-cccc-22087fb081db",
+            },
+            {
+              config_hash = "a9a166c59873245db8f1a747ba9a80a7",
+              hostname = "localhost",
+              ip = "127.0.0.1",
+              cert_details = {
+                expiry_timestamp = 1897136778,
+              }
+            }
+          )
+
+        assert.is_truthy(p)
+        assert.is_nil(err)
+      end)
+
+      it(":update()", function()
+        -- this time update instead of insert
+        local p, err =
+          db.clustering_data_planes:update(
+            {
+              id = "eb51145a-aaaa-bbbb-cccc-22087fb081db",
+            },
+            {
+              config_hash = "a9a166c59873245db8f1a747ba9a80a7",
+              cert_details = {
+                expiry_timestamp = 1888983905,
+              }
+            }
+          )
+
+        assert.is_truthy(p)
+        assert.is_nil(err)
+      end)
+    end)
   end) -- kong.db [strategy]
 end
diff --git a/spec/02-integration/09-hybrid_mode/01-sync_spec.lua b/spec/02-integration/09-hybrid_mode/01-sync_spec.lua
@@ -784,4 +784,120 @@ describe("CP/DP labels #" .. strategy, function()
   end)
 end)
 
+describe("CP/DP cert details(cluster_mtls = shared) #" .. strategy, function()
+  lazy_setup(function()
+    helpers.get_db_utils(strategy) -- runs migrations
+
+    assert(helpers.start_kong({
+      role = "control_plane",
+      cluster_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_cert_key = "spec/fixtures/kong_clustering.key",
+      database = strategy,
+      db_update_frequency = 0.1,
+      cluster_listen = "127.0.0.1:9005",
+      nginx_conf = "spec/fixtures/custom_nginx.template",
+    }))
+
+    assert(helpers.start_kong({
+      role = "data_plane",
+      database = "off",
+      prefix = "servroot2",
+      cluster_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_cert_key = "spec/fixtures/kong_clustering.key",
+      cluster_control_plane = "127.0.0.1:9005",
+      proxy_listen = "0.0.0.0:9002",
+      nginx_conf = "spec/fixtures/custom_nginx.template",
+      cluster_dp_labels="deployment:mycloud,region:us-east-1",
+    }))
+  end)
+
+  lazy_teardown(function()
+    helpers.stop_kong("servroot2")
+    helpers.stop_kong()
+  end)
+
+  describe("status API", function()
+    it("shows DP cert details", function()
+      helpers.wait_until(function()
+        local admin_client = helpers.admin_client()
+        finally(function()
+          admin_client:close()
+        end)
+
+        local res = assert(admin_client:get("/clustering/data-planes"))
+        local body = assert.res_status(200, res)
+        local json = cjson.decode(body)
+
+        for _, v in pairs(json.data) do
+          if v.ip == "127.0.0.1" then
+            assert.equal(1888983905, v.cert_details.expiry_timestamp)
+            return true
+          end
+        end
+      end, 3)
+    end)
+  end)
+end)
+
+describe("CP/DP cert details(cluster_mtls = pki) #" .. strategy, function()
+  lazy_setup(function()
+    helpers.get_db_utils(strategy) -- runs migrations
+
+    assert(helpers.start_kong({
+      role = "control_plane",
+      cluster_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_cert_key = "spec/fixtures/kong_clustering.key",
+      db_update_frequency = 0.1,
+      database = strategy,
+      cluster_listen = "127.0.0.1:9005",
+      nginx_conf = "spec/fixtures/custom_nginx.template",
+      -- additional attributes for PKI:
+      cluster_mtls = "pki",
+      cluster_ca_cert = "spec/fixtures/kong_clustering_ca.crt",
+    }))
+
+    assert(helpers.start_kong({
+      role = "data_plane",
+      nginx_conf = "spec/fixtures/custom_nginx.template",
+      database = "off",
+      prefix = "servroot2",
+      cluster_cert = "spec/fixtures/kong_clustering_client.crt",
+      cluster_cert_key = "spec/fixtures/kong_clustering_client.key",
+      cluster_control_plane = "127.0.0.1:9005",
+      proxy_listen = "0.0.0.0:9002",
+      -- additional attributes for PKI:
+      cluster_mtls = "pki",
+      cluster_server_name = "kong_clustering",
+      cluster_ca_cert = "spec/fixtures/kong_clustering.crt",
+    }))
+  end)
+
+  lazy_teardown(function()
+    helpers.stop_kong("servroot2")
+    helpers.stop_kong()
+  end)
+
+  describe("status API", function()
+    it("shows DP cert details", function()
+      helpers.wait_until(function()
+        local admin_client = helpers.admin_client()
+        finally(function()
+          admin_client:close()
+        end)
+
+        local res = admin_client:get("/clustering/data-planes")
+        local body = assert.res_status(200, res)
+        local json = cjson.decode(body)
+
+        for _, v in pairs(json.data) do
+          if v.ip == "127.0.0.1" then
+            assert.equal(1897136778, v.cert_details.expiry_timestamp)
+            return true
+          end
+        end
+      end, 3)
+    end)
+  end)
+end)
+
 end
diff --git a/spec/05-migration/db/migrations/core/022_350_to_360_spec.lua b/spec/05-migration/db/migrations/core/022_350_to_360_spec.lua
@@ -0,0 +1,7 @@
+local uh = require "spec/upgrade_helpers"
+
+describe("database migration", function()
+  uh.old_after_up("has created the expected new columns", function()
+    assert.table_has_column("clustering_data_planes", "cert_details", "jsonb")
+  end)
+end)