Merge branch 'master' into oauth-mtls-policy

CHANGELOG.md

@@ -10,14 +10,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 - Introduce possibility of specifying policy order restrictions in their schemas. APIcast now shows a warning when those restrictions are not respected [#1088](https://github.com/3scale/APIcast/pull/1088), [THREESCALE-2896](https://issues.jboss.org/browse/THREESCALE-2896)
 - Added new parameters to logging policy to allow custom access log [PR #1081](https://github.com/3scale/APIcast/pull/1089) [THREESCALE-1234](https://issues.jboss.org/browse/THREESCALE-1234)[THREESCALE-2876](https://issues.jboss.org/browse/THREESCALE-2876)
+- Added http_proxy policy to use an HTTP proxy in api_backed calls. [THREESCALE-2696](https://issues.jboss.org/browse/THREESCALE-2696) [PR #1080](https://github.com/3scale/APIcast/pull/1080)
+- Option to load service configurations one by one lazily [PR #1099](https://github.com/3scale/APIcast/pull/1099)
+- New maintenance mode policy, useful for maintenance periods. [PR #1105](https://github.com/3scale/APIcast/pull/1105), [THREESCALE-3189](https://issues.jboss.org/browse/THREESCALE-3189)
+- Remove dnsmasq process for APIcast [PR #1090](https://github.com/3scale/APIcast/pull/1090), [THREESCALE-1555](https://issues.jboss.org/browse/THREESCALE-1555)
+- Allow to use capture function in liquid templates. [PR #1107](https://github.com/3scale/APIcast/pull/1107), [THREESCALE-1911](https://issues.jboss.org/browse/THREESCALE-1911)
 - OAuth 2.0 MTLS policy [PR #1101](https://github.com/3scale/APIcast/pull/1101) [Issue #1003](https://github.com/3scale/APIcast/issues/1003)
 
+
 ## [3.6.0-rc1] - 2019-07-04
 
 ### Added
 
 - Extended variables in Liquid template operations [PR #1081](https://github.com/3scale/APIcast/pull/1081)[THREESCALE-2927](https://issues.jboss.org/browse/THREESCALE-2927)
 
+
 ## [3.6.0-beta1] - 2019-06-18
 
 ### Added

doc/parameters.md

@@ -51,6 +51,29 @@ Double colon (`:`) separated list of environments (or paths) APIcast should load
 It can be used instead of `-e` or `---environment` parameter on the CLI and for example
 stored in the container image as default environment. Any value passed on the CLI overrides this variable.
 
+### `APICAST_LOAD_SERVICES_WHEN_NEEDED`
+**Values:**
+- `true` or `1` for _true_
+- `false`, `0` or empty for _false_
+
+**Default:** _false_
+
+This option can be used when there are many services configured. However, its
+performance depends on additional factors such as the number of services, the
+latency between APIcast and the 3scale Admin Portal, the Time To Live (TTL) of
+the configuration, etc.
+
+By default, APIcast loads all the services each time it downloads its
+configuration from the Admin Portal. With a large number of services, this could
+become problematic. When this option is enabled, the configurations are loaded
+lazily. APIcast will only load the ones configured for the host specified in the
+host header of the request.
+
+Notes:
+- The caching defined by `APICAST_CONFIGURATION_CACHE` applies.
+- This option will be disabled when `APICAST_CONFIGURATION_LOADER` is `boot`.
+- Not compatible with `APICAST_PATH_ROUTING`.
+
 ### `APICAST_LOG_FILE`
 
 **Default:** _stderr_

gateway/.s2i/bin/run

@@ -18,12 +18,4 @@ else
   apicast=apicast
 fi
 
-dnsmasq --listen-address=127.0.0.1 --port=5353 \
-  --all-servers --no-host --no-hosts \
-  --cache-size=1000 --no-negcache --domain-needed \
-  --server="${RESOLVER:-}" \
-  --log-facility=- ${DNSMASQ_OPTIONS:-} \
-
-export RESOLVER=127.0.0.1:5353
-
 exec "${apicast}" "$@"

gateway/src/apicast/configuration_loader/remote_v2.lua

@@ -6,6 +6,7 @@ local insert = table.insert
 local rawset = rawset
 local encode_args = ngx.encode_args
 local tonumber = tonumber
+local pcall = pcall
 
 local tablex = require('pl.tablex')
 local deepcopy = tablex.deepcopy
@@ -77,6 +78,77 @@ local function array()
   return setmetatable({}, cjson.empty_array_mt)
 end
 
+local function services_index_endpoint(portal_endpoint)
+  return resty_url.join(portal_endpoint, '/admin/api/services.json')
+end
+
+local function service_config_endpoint(portal_endpoint, service_id, env, version)
+  local version_override = resty_env.get(
+      format('APICAST_SERVICE_%s_CONFIGURATION_VERSION', service_id)
+  )
+
+  return resty_url.join(
+      portal_endpoint,
+      '/admin/api/services/', service_id , '/proxy/configs/', env, '/',
+      format('%s.json', version_override or version)
+  )
+end
+
+local function endpoint_for_services_with_host(portal_endpoint, env, host)
+  local query_args = encode_args({ host = host })
+
+  return format(
+      "%s/admin/api/services/proxy/configs/%s.json?%s",
+      portal_endpoint,
+      env,
+      query_args
+  )
+end
+
+local function parse_resp_body(self, resp_body)
+  local ok, res = pcall(cjson.decode, resp_body)
+  if not ok then return nil, res end
+  local json = res
+
+  local config = { services = array(), oidc = array() }
+
+  local proxy_configs = json.proxy_configs or {}
+
+  for i, proxy_conf in ipairs(proxy_configs) do
+    local proxy_config = proxy_conf.proxy_config
+
+    -- Copy the config because parse_service have side-effects. It adds
+    -- liquid templates in some policies and those cannot be encoded into a
+    -- JSON. We should get rid of these side effects.
+    local original_proxy_config = deepcopy(proxy_config)
+
+    local service = configuration.parse_service(proxy_config.content)
+    local oidc = self:oidc_issuer_configuration(service)
+
+    -- Assign false instead of nil to avoid sparse arrays. cjson raises an
+    -- error by default when converting sparse arrays.
+    config.oidc[i] = oidc or false
+
+    config.services[i] = original_proxy_config.content
+  end
+
+  return cjson.encode(config)
+end
+
+local function load_just_the_services_needed()
+  return resty_env.enabled('APICAST_LOAD_SERVICES_WHEN_NEEDED') and
+         resty_env.value('APICAST_CONFIGURATION_LOADER') == 'lazy'
+end
+
+-- When the APICAST_LOAD_SERVICES_WHEN_NEEDED is enabled, but the config loader
+-- is boot, APICAST_LOAD_SERVICES_WHEN_NEEDED is going to be ignored. But in
+-- that case, env refers to a host and we need to reset it to pick the env
+-- again.
+local function reset_env()
+  return resty_env.enabled('APICAST_LOAD_SERVICES_WHEN_NEEDED') and
+         resty_env.value('APICAST_CONFIGURATION_LOADER') == 'boot'
+end
+
 function _M:index(host)
   local http_client = self.http_client
 
@@ -107,47 +179,42 @@ function _M:index(host)
   ngx.log(ngx.DEBUG, 'index get status: ', res.status, ' url: ', url)
 
   if res.status == 200 then
-    local json = cjson.decode(res.body)
-
-    local config = { services = array(), oidc = array() }
-
-    local proxy_configs = json.proxy_configs or {}
-
-    for i=1, #proxy_configs do
-      local proxy_config = proxy_configs[i].proxy_config
-
-      -- Copy the config because parse_service have side-effects. It adds
-      -- liquid templates in some policies and those cannot be encoded into a
-      -- JSON. We should get rid of these side effects.
-      local original_proxy_config = deepcopy(proxy_config)
-
-      local service = configuration.parse_service(proxy_config.content)
-      local oidc = self:oidc_issuer_configuration(service)
+    return parse_resp_body(self, res.body)
+  else
+    return nil, 'invalid status'
+  end
+end
 
-      -- Assign false instead of nil to avoid sparse arrays. cjson raises an
-      -- error by default when converting sparse arrays.
-      config.oidc[i] = oidc or false
+function _M:load_configs_for_env_and_host(env, host)
+  local url = endpoint_for_services_with_host(self.endpoint, env, host)
 
-      config.services[i] = original_proxy_config.content
-    end
+  local response = self.http_client.get(url)
 
-    return cjson.encode(config)
+  if response.status == 200 then
+    return parse_resp_body(self, response.body)
   else
-    return nil, 'invalid status'
+    ngx.log(ngx.ERR, 'failed to load proxy configs')
+    return false
   end
 end
 
 function _M:call(environment)
+  local load_just_for_host = load_just_the_services_needed()
+
   if self == _M  or not self then
     local host = environment
     local m = _M.new()
     local ret, err = m:index(host)
 
-    if not ret then
-      return m:call()
-    else
+    if ret then
       return ret, err
     end
+
+    if load_just_for_host then
+      return m:call(host)
+    else
+      return m:call()
+    end
   end
 
   local http_client = self.http_client
@@ -156,7 +223,16 @@ function _M:call(environment)
     return nil, 'not initialized'
   end
 
+  if load_just_for_host then
+    return self:load_configs_for_env_and_host(resty_env.value('THREESCALE_DEPLOYMENT_ENV'), environment)
+  end
+
   local env = environment or resty_env.value('THREESCALE_DEPLOYMENT_ENV')
+
+  if reset_env() then
+    env = resty_env.value('THREESCALE_DEPLOYMENT_ENV')
+  end
+
   if not env then
     return nil, 'missing environment'
   end
@@ -181,12 +257,12 @@ function _M:call(environment)
     end
   end
 
-  for i=1, #configs do
-    configs.services[i] = configs[i].content
+  for i, conf in ipairs(configs) do
+    configs.services[i] = conf.content
 
     -- Assign false instead of nil to avoid sparse arrays. cjson raises an
     -- error by default when converting sparse arrays.
-    configs.oidc[i] = configs[i].oidc or false
+    configs.oidc[i] = conf.oidc or false
 
     configs[i] = nil
   end
@@ -206,6 +282,18 @@ local services_subset = function()
   end
 end
 
+-- Returns a table with services.
+-- There are 2 cases:
+-- A) with APICAST_SERVICES_LIST. The method returns a table where each element
+--    contains a single field "service", which is another table with just one
+--    element: "id".
+--    Example: { { service = { id = 123 } }, { service = { id = 456 } } }
+-- B) without APICAST_SERVICES_LIST. The services are fetched from an endpoint
+--    in Porta: https://ACCESS-TOKEN@ADMIN-DOMAIN/admin/api/services.json
+--    The function returns the services decoded as Lua tables.
+--    Each element follows the same format described above, but instead of
+--    having just "id", there are other fields: "backend_version",
+--    "created_at", "state", "system_name", and other fields.
 function _M:services()
   local services = services_subset()
   if services then return services end
@@ -222,7 +310,7 @@ function _M:services()
     return nil, 'no endpoint'
   end
 
-  local url = resty_url.join(self.endpoint, '/admin/api/services.json')
+  local url = services_index_endpoint(endpoint)
   local res, err = http_client.get(url)
 
   if not res and err then
@@ -259,13 +347,7 @@ function _M:config(service, environment, version)
   if not environment then return nil, 'missing environment' end
   if not version then return nil, 'missing version' end
 
-  local version_override = resty_env.get(format('APICAST_SERVICE_%s_CONFIGURATION_VERSION', id))
-
-  local url = resty_url.join(
-    endpoint,
-    '/admin/api/services/', id , '/proxy/configs/', environment, '/',
-    format('%s.json', version_override or version)
-  )
+  local url = service_config_endpoint(endpoint, id, environment, version)
 
   local res, err = http_client.get(url)
 

gateway/src/apicast/http_proxy.lua

@@ -75,7 +75,7 @@ local function absolute_url(uri)
             uri.scheme,
             host,
             port,
-            uri.path or ''
+            uri.path or '/'
     )
 end
 
@@ -97,6 +97,7 @@ local function forward_https_request(proxy_uri, uri)
         -- In POST requests with HTTPS, the result of that call is nil, and it
         -- results in a time-out.
         body = ngx.req.get_body_data(),
+        proxy_uri = proxy_uri
     }
 
     local httpc, err = http_proxy.new(request)
@@ -138,8 +139,12 @@ function _M.request(upstream, proxy_uri)
     local uri = upstream.uri
 
     if uri.scheme == 'http' then -- rewrite the request to use http_proxy
+        local err
         upstream:use_host_header(uri.host) -- to keep correct Host header in case we need to resolve it to IP
-        upstream.servers = resolve_servers(proxy_uri)
+        upstream.servers, err = resolve_servers(proxy_uri)
+        if err then
+          ngx.log(ngx.WARN, "HTTP proxy is set, but no servers have been resolved, err: ", err)
+        end
         upstream.uri.path = absolute_url(uri)
         upstream:rewrite_request()
         return