Support http proxy configuration via env variables (#2850)

lib/livebook/application.ex

@@ -2,6 +2,8 @@ defmodule Livebook.Application do
   use Application
 
   def start(_type, _args) do
+    Livebook.Utils.HTTP.set_proxy_options()
+
     Livebook.ZTA.init()
     create_teams_hub = parse_teams_hub()
     setup_optional_dependencies()

lib/livebook/fly_api.ex

@@ -250,7 +250,9 @@ defmodule Livebook.FlyAPI do
       |> Keyword.merge(opts)
       |> Keyword.merge(test_options())
 
-    case Req.request(opts) do
+    req = opts |> Req.new() |> Livebook.Utils.req_attach_defaults()
+
+    case Req.request(req) do
       {:ok, %{status: status, body: body}} when status in 200..299 ->
         {:ok, body}
 
@@ -281,7 +283,9 @@ defmodule Livebook.FlyAPI do
       ]
       |> Keyword.merge(test_options())
 
-    case Req.request(opts) do
+    req = opts |> Req.new() |> Livebook.Utils.req_attach_defaults()
+
+    case Req.request(req) do
       {:ok, %{status: 200, body: body}} ->
         case body do
           %{"errors" => [%{"extensions" => %{"code" => "UNAUTHORIZED"}} | _]} ->

lib/livebook/k8s_api.ex

@@ -91,7 +91,7 @@ defmodule Livebook.K8sAPI do
   """
   @spec watch_pod_events(kubeconfig(), String.t(), String.t()) :: {:ok, Enumerable.t()} | error()
   def watch_pod_events(kubeconfig, namespace, name) do
-    req = Req.new() |> Kubereq.attach(kubeconfig: kubeconfig, api_version: "v1", kind: "Event")
+    req = build_req() |> Kubereq.attach(kubeconfig: kubeconfig, api_version: "v1", kind: "Event")
 
     Kubereq.watch(req, namespace,
       field_selectors: [
@@ -120,7 +120,7 @@ defmodule Livebook.K8sAPI do
         when data: list(%{name: String.t()})
   def list_namespaces(kubeconfig) do
     req =
-      Req.new() |> Kubereq.attach(kubeconfig: kubeconfig, api_version: "v1", kind: "Namespace")
+      build_req() |> Kubereq.attach(kubeconfig: kubeconfig, api_version: "v1", kind: "Namespace")
 
     case Kubereq.list(req, nil) do
       {:ok, %{status: 200, body: %{"items" => items}}} ->
@@ -198,7 +198,7 @@ defmodule Livebook.K8sAPI do
         when data: list(%{name: String.t()})
   def list_storage_classes(kubeconfig) do
     req =
-      Req.new()
+      build_req()
       |> Kubereq.attach(
         kubeconfig: kubeconfig,
         api_version: "storage.k8s.io/v1",
@@ -260,7 +260,7 @@ defmodule Livebook.K8sAPI do
     }
 
     req =
-      Req.new()
+      build_req()
       |> Kubereq.attach(
         kubeconfig: kubeconfig,
         api_version: "authorization.k8s.io/v1",
@@ -304,11 +304,15 @@ defmodule Livebook.K8sAPI do
   end
 
   defp pod_req(kubeconfig) do
-    Req.new() |> Kubereq.attach(kubeconfig: kubeconfig, api_version: "v1", kind: "Pod")
+    build_req() |> Kubereq.attach(kubeconfig: kubeconfig, api_version: "v1", kind: "Pod")
   end
 
   defp pvc_req(kubeconfig) do
-    Req.new()
+    build_req()
     |> Kubereq.attach(kubeconfig: kubeconfig, api_version: "v1", kind: "PersistentVolumeClaim")
   end
+
+  defp build_req() do
+    Req.new() |> Livebook.Utils.req_attach_defaults()
+  end
 end

lib/livebook/runtime/dependencies.ex

@@ -6,18 +6,10 @@ defmodule Livebook.Runtime.Dependencies do
           {:ok, String.t()} | {:error, String.t()}
   def add_dependencies(code, dependencies) do
     deps = Enum.map(dependencies, & &1.dep)
-    config = Enum.reduce(dependencies, [], &deep_merge(&2, &1.config))
+    config = Enum.reduce(dependencies, [], &Livebook.Utils.keyword_deep_merge(&2, &1.config))
     add_mix_deps(code, deps, config)
   end
 
-  defp deep_merge(left, right) do
-    if Keyword.keyword?(left) and Keyword.keyword?(right) do
-      Keyword.merge(left, right, fn _key, left, right -> deep_merge(left, right) end)
-    else
-      right
-    end
-  end
-
   @doc """
   Finds or adds a `Mix.install/2` call to `code` and modifies it to
   include the given Mix deps.

lib/livebook/teams/requests.ex

@@ -294,9 +294,9 @@ defmodule Livebook.Teams.Requests do
   defp build_req() do
     Req.new(
       base_url: Livebook.Config.teams_url(),
-      inet6: String.ends_with?(Livebook.Config.teams_url(), ".flycast"),
       headers: [{"x-lb-version", Livebook.Config.app_version()}]
     )
+    |> Livebook.Utils.req_attach_defaults()
   end
 
   defp add_team_auth(req, nil), do: req

lib/livebook/teams/web_socket.ex

@@ -22,19 +22,9 @@ defmodule Livebook.Teams.WebSocket do
     {http_scheme, ws_scheme} = parse_scheme(uri)
     state = %{status: nil, headers: [], body: []}
 
-    transport_opts =
-      if http_scheme == :https do
-        [cacerts: :public_key.cacerts_get()]
-      else
-        []
-      end
-
-    transport_opts =
-      if String.ends_with?(Livebook.Config.teams_url(), ".flycast"),
-        do: Keyword.put(transport_opts, :inet6, true),
-        else: transport_opts
-
-    opts = [protocols: [:http1], transport_opts: transport_opts]
+    opts = Livebook.Utils.mint_connect_options_for_uri(uri)
+
+    opts = Keyword.merge(opts, protocols: [:http1])
 
     with {:ok, conn} <- Mint.HTTP.connect(http_scheme, uri.host, uri.port, opts),
          {:ok, conn, ref} <- Mint.WebSocket.upgrade(ws_scheme, conn, @ws_path, headers) do

lib/livebook/utils.ex

@@ -204,6 +204,18 @@ defmodule Livebook.Utils do
     end
   end
 
+  @doc """
+  Recursively merges keyword lists.
+  """
+  @spec keyword_deep_merge(keyword(), keyword()) :: keyword()
+  def keyword_deep_merge(left, right) do
+    if Keyword.keyword?(left) and Keyword.keyword?(right) do
+      Keyword.merge(left, right, fn _key, left, right -> keyword_deep_merge(left, right) end)
+    else
+      right
+    end
+  end
+
   @doc """
   Validates if the given URL is syntactically valid.
 
@@ -745,4 +757,66 @@ defmodule Livebook.Utils do
   def ip_to_host({0, 0, 0, 0}), do: "localhost"
   def ip_to_host({127, 0, 0, 1}), do: "localhost"
   def ip_to_host(ip), do: ip |> :inet.ntoa() |> List.to_string()
+
+  @doc """
+  Req plugin adding global Livebook-specific plugs.
+
+  The plugin covers cacerts and HTTP proxy configuration.
+  """
+  @spec req_attach_defaults(Req.Request.t()) :: Req.Request.t()
+  def req_attach_defaults(req) do
+    Req.Request.append_request_steps(req,
+      connect_options: fn request ->
+        uri = URI.parse(request.url)
+        connect_options = mint_connect_options_for_uri(uri)
+        Req.Request.merge_options(request, connect_options: connect_options)
+      end
+    )
+  end
+
+  @doc """
+  Returns options for `Mint.HTTP.connect/4` that should be used for
+  the given target URI.
+  """
+  @spec mint_connect_options_for_uri(URI.t()) :: keyword()
+  def mint_connect_options_for_uri(uri) do
+    http_proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy")
+    https_proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") || http_proxy
+
+    no_proxy =
+      if no_proxy = System.get_env("NO_PROXY") || System.get_env("no_proxy") do
+        String.split(no_proxy, ",")
+      else
+        []
+      end
+
+    proxy_opts =
+      cond do
+        uri.host in no_proxy -> []
+        uri.scheme == "http" && http_proxy -> proxy_options(http_proxy)
+        uri.scheme == "https" && https_proxy -> proxy_options(https_proxy)
+        true -> []
+      end
+
+    cacertfile = Livebook.Config.cacertfile()
+
+    cert_opts =
+      if uri.scheme == "https" && cacertfile do
+        [transport_opts: [cacertfile: cacertfile]]
+      else
+        []
+      end
+
+    proxy_opts ++ cert_opts
+  end
+
+  defp proxy_options(proxy) do
+    uri = URI.parse(proxy || "")
+
+    if uri.host && uri.port do
+      [proxy: {:http, uri.host, uri.port, []}]
+    else
+      []
+    end
+  end
 end

lib/livebook/utils/http.ex

@@ -212,4 +212,31 @@ defmodule Livebook.Utils.HTTP do
       ]
     ]
   end
+
+  @doc false
+  def set_proxy_options() do
+    http_proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy")
+    https_proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy")
+
+    no_proxy =
+      if no_proxy = System.get_env("NO_PROXY") || System.get_env("no_proxy") do
+        no_proxy
+        |> String.split(",")
+        |> Enum.map(&String.to_charlist/1)
+      else
+        []
+      end
+
+    set_proxy_option(:proxy, http_proxy, no_proxy)
+    set_proxy_option(:https_proxy, https_proxy, no_proxy)
+  end
+
+  defp set_proxy_option(proxy_scheme, proxy, no_proxy) do
+    uri = URI.parse(proxy || "")
+
+    if uri.host && uri.port do
+      host = String.to_charlist(uri.host)
+      :httpc.set_options([{proxy_scheme, {{host, uri.port}, no_proxy}}])
+    end
+  end
 end