Add option :return_errors to ParallelCompiler (elixir-lang#6446)

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -13,7 +13,7 @@ defmodule Kernel.ParallelCompiler do
 
   If there is an error during compilation or if `warnings_as_errors`
   is set to `true` and there is a warning, this function will fail
-  with an exception.
+  with an exception unless the option `:return_errors` is set to true.
 
   This function accepts the following options:
 
@@ -38,7 +38,14 @@ defmodule Kernel.ParallelCompiler do
       they are loaded into memory. If you want a file to actually be written to
       `dest`, use `files_to_path/3` instead.
 
-  Returns the modules generated by each compiled file.
+    * `:return_errors` - return `{:ok, modules, warnings}` on success or
+      `{:error, errors, warnings}` if compilation fails or if there are warnings and
+      `warnings_as_errors` is set to true.
+
+  When the `:return_errors` option is not set to true, this function returns
+  the modules generated by each compiled file and will fail with an
+  exception if compilation fails. When `:return_errors` is true, it returns
+  `{:ok, modules, warnings}` on success and `{:error, errors, warnings}` on failure.
   """
   def files(files, options \\ [])
 
@@ -71,16 +78,28 @@ defmodule Kernel.ParallelCompiler do
       queued: [],
       schedulers: schedulers,
       result: [],
+      warnings: [],
     })
 
     # In case --warning-as-errors is enabled and there was a warning,
     # compilation status will be set to error.
-    case :elixir_code_server.call({:compilation_status, compiler_pid}) do
-      :ok ->
-        result
-      :error ->
-        IO.puts :stderr, "Compilation failed due to warnings while using the --warnings-as-errors option"
-        exit({:shutdown, 1})
+    compilation_status = :elixir_code_server.call({:compilation_status, compiler_pid})
+    result =
+      case {result, compilation_status} do
+        {{:ok, _, warnings}, :error} ->
+          IO.puts :stderr, "Compilation failed due to warnings while using the --warnings-as-errors option"
+          {:error, warnings, []}
+        _ ->
+          result
+      end
+
+    if Keyword.get(options, :return_errors) do
+      result
+    else
+      case result do
+        {:ok, modules, _} -> modules
+        {:error, _, _} -> exit({:shutdown, 1})
+      end
     end
   end
 
@@ -138,12 +157,14 @@ defmodule Kernel.ParallelCompiler do
   end
 
   # No more files, nothing waiting, queue is empty, we are done
-  defp spawn_compilers(%{entries: [], waiting: [], queued: [], result: result}) do
-    for {:module, mod} <- result, do: mod
+  defp spawn_compilers(%{entries: [], waiting: [], queued: [], result: result, warnings: warnings}) do
+    modules = for {:module, mod} <- result, do: mod
+    warnings = Enum.reverse(warnings)
+    {:ok, modules, warnings}
   end
 
   # Queued x, waiting for x: POSSIBLE ERROR! Release processes so we get the failures
-  defp spawn_compilers(%{entries: [], waiting: waiting, queued: queued} = state) when length(waiting) == length(queued) do
+  defp spawn_compilers(%{entries: [], waiting: waiting, queued: queued, warnings: warnings} = state) when length(waiting) == length(queued) do
     entries = for {pid, _, _, _} <- queued,
                   entry = waiting_on_without_definition(waiting, pid),
                   {_, _, ref, on, _} = entry,
@@ -162,8 +183,11 @@ defmodule Kernel.ParallelCompiler do
     |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))
     |> Enum.sort_by(&length(elem(&1, 1)))
     |> case do
-      [{_on, refs} | _] -> spawn_compilers(%{state | entries: refs})
-      [] -> handle_deadlock(waiting, queued)
+      [{_on, refs} | _] ->
+        spawn_compilers(%{state | entries: refs})
+      [] ->
+        errors = handle_deadlock(waiting, queued)
+        {:error, errors, warnings}
     end
   end
 
@@ -183,7 +207,7 @@ defmodule Kernel.ParallelCompiler do
 
   # Wait for messages from child processes
   defp wait_for_messages(state) do
-    %{entries: entries, options: options, waiting: waiting, queued: queued, result: result} = state
+    %{entries: entries, options: options, waiting: waiting, queued: queued, result: result, warnings: warnings} = state
 
     receive do
       {:struct_available, module} ->
@@ -234,10 +258,15 @@ defmodule Kernel.ParallelCompiler do
         spawn_compilers(state)
 
       {:warning, file, line, message} ->
+        file = if file, do: Path.absname(file), else: nil
+        message = :unicode.characters_to_binary(message)
+
         if callback = Keyword.get(options, :each_warning) do
           callback.(file, line, message)
         end
-        wait_for_messages(state)
+
+        warning = {file, line, message}
+        wait_for_messages(%{state | warnings: [warning | state.warnings]})
 
       {:file_compiled, child_pid, file, :ok} ->
         discard_down(child_pid)
@@ -259,10 +288,13 @@ defmodule Kernel.ParallelCompiler do
         discard_down(child_pid)
         print_error(file, kind, reason, stack)
         terminate(queued)
+        {:error, [to_error(file, kind, reason, stack)], warnings}
 
       {:DOWN, ref, :process, _pid, reason} ->
-        handle_down(queued, ref, reason)
-        wait_for_messages(state)
+        case handle_down(queued, ref, reason) do
+          :ok -> wait_for_messages(state)
+          {:error, errors} -> {:error, errors, warnings}
+        end
     end
   end
 
@@ -280,6 +312,7 @@ defmodule Kernel.ParallelCompiler do
       {_child, ^ref, file, _timer_ref} ->
         print_error(file, :exit, reason, [])
         terminate(queued)
+        {:error, [to_error(file, :exit, reason, [])]}
       _ ->
         :ok
     end
@@ -292,11 +325,11 @@ defmodule Kernel.ParallelCompiler do
         Process.exit(pid, :kill)
 
         {_kind, ^pid, _, on, _} = List.keyfind(waiting, pid, 1)
-        error = CompileError.exception(description: "deadlocked waiting on module #{inspect on}",
-                                       file: nil, line: nil)
+        description = "deadlocked waiting on module #{inspect on}"
+        error = CompileError.exception(description: description, file: nil, line: nil)
         print_error(file, :error, error, stacktrace)
 
-        {file, on}
+        {file, on, description}
       end
 
     IO.puts """
@@ -310,19 +343,19 @@ defmodule Kernel.ParallelCompiler do
       |> Enum.map(& &1 |> elem(0) |> String.length)
       |> Enum.max
 
-    for {file, mod} <- deadlock do
+    for {file, mod, _} <- deadlock do
       IO.puts ["  ", String.pad_leading(file, max), " => " | inspect(mod)]
     end
 
     IO.puts ""
-    exit({:shutdown, 1})
+
+    for {file, _, description} <- deadlock, do: {Path.absname(file), nil, description}
   end
 
   defp terminate(queued) do
     for {pid, _, _, _} <- queued do
       Process.exit(pid, :kill)
     end
-    exit({:shutdown, 1})
   end
 
   defp print_error(file, kind, reason, stack) do
@@ -345,4 +378,11 @@ defmodule Kernel.ParallelCompiler do
         :ok
     end
   end
+
+  defp to_error(file, kind, reason, stack) do
+    file = Path.absname(file)
+    line = Map.get(reason, :line)
+    message = :unicode.characters_to_binary(Kernel.CLI.format_error(kind, reason, stack))
+    {file, line, message}
+  end
 end

lib/elixir/src/elixir_errors.erl

@@ -12,24 +12,13 @@
 warn(none, File, Warning) ->
   warn(0, File, Warning);
 warn(Line, File, Warning) when is_integer(Line), is_binary(File) ->
-  CompilerPid = get(elixir_compiler_pid),
-  if
-    CompilerPid =/= undefined ->
-      CompilerPid ! {warning, File, Line, Warning};
-    true -> ok
-  end,
-  warn([Warning, "\n  ", file_format(Line, File), $\n]).
+  send_warning(File, Line, Warning),
+  print_warning([Warning, "\n  ", file_format(Line, File), $\n]).
 
 -spec warn(unicode:chardata()) -> ok.
 warn(Message) ->
-  CompilerPid = get(elixir_compiler_pid),
-  if
-    CompilerPid =/= undefined ->
-      elixir_code_server:cast({register_warning, CompilerPid});
-    true -> ok
-  end,
-  io:put_chars(standard_error, [warning_prefix(), Message, $\n]),
-  ok.
+  send_warning(nil, nil, Message),
+  print_warning(Message).
 
 warning_prefix() ->
   case application:get_env(elixir, ansi_enabled) of
@@ -122,6 +111,20 @@ parse_erl_term(Term) ->
 
 %% Helpers
 
+print_warning(Message) ->
+  io:put_chars(standard_error, [warning_prefix(), Message, $\n]),
+  ok.
+
+send_warning(File, Line, Message) ->
+  CompilerPid = get(elixir_compiler_pid),
+  if
+    CompilerPid =/= undefined ->
+      CompilerPid ! {warning, File, Line, Message},
+      elixir_code_server:cast({register_warning, CompilerPid});
+    true -> ok
+  end,
+  ok.
+
 file_format(0, File) ->
   io_lib:format("~ts", [elixir_utils:relative_to_cwd(File)]);
 

lib/elixir/test/elixir/kernel/parallel_compiler_test.exs

@@ -9,7 +9,7 @@ defmodule Kernel.ParallelCompilerTest do
   test "compiles files solving dependencies" do
     fixtures = [fixture_path("parallel_compiler/bar.ex"), fixture_path("parallel_compiler/foo.ex")]
     assert capture_io(fn ->
-      assert [BarParallel, FooParallel] = Kernel.ParallelCompiler.files fixtures
+      assert {:ok, [BarParallel, FooParallel], []} = Kernel.ParallelCompiler.files(fixtures, return_errors: true)
     end) =~ "message_from_foo"
   after
     Enum.map [FooParallel, BarParallel], fn mod ->
@@ -20,34 +20,51 @@ defmodule Kernel.ParallelCompilerTest do
 
   test "compiles files with structs solving dependencies" do
     fixtures = [fixture_path("parallel_struct/bar.ex"), fixture_path("parallel_struct/foo.ex")]
-    assert [BarStruct, FooStruct] = Kernel.ParallelCompiler.files(fixtures) |> Enum.sort
+    assert {:ok, modules, []} = Kernel.ParallelCompiler.files(fixtures, return_errors: true)
+    assert [BarStruct, FooStruct] = Enum.sort(modules)
   after
     Enum.map [FooStruct, BarStruct], fn mod ->
       :code.purge(mod)
       :code.delete(mod)
     end
   end
 
-  test "emits struct undefined error when local struct is undefined" do
-    fixtures = [fixture_path("parallel_struct/undef.ex")]
+  test "returns struct undefined error when local struct is undefined" do
+    fixture = fixture_path("parallel_struct/undef.ex")
+    expected_msg = "Undef.__struct__/1 is undefined, cannot expand struct Undef"
     assert capture_io(fn ->
-      assert catch_exit(Kernel.ParallelCompiler.files(fixtures)) == {:shutdown, 1}
-    end) =~ "Undef.__struct__/1 is undefined, cannot expand struct Undef"
+      assert {:error, [{^fixture, 3, msg}], []} = Kernel.ParallelCompiler.files([fixture], return_errors: true)
+      assert msg =~ expected_msg
+    end) =~ expected_msg
+  end
+
+  test "exits on error if :return_errors is false" do
+    fixture = fixture_path("parallel_struct/undef.ex")
+    capture_io(fn ->
+      assert {:shutdown, 1} = catch_exit(Kernel.ParallelCompiler.files([fixture]))
+    end)
   end
 
   test "does not hang on missing dependencies" do
-    fixtures = [fixture_path("parallel_compiler/bat.ex")]
+    fixture = fixture_path("parallel_compiler/bat.ex")
+    expected_msg = "ThisModuleWillNeverBeAvailable.__struct__/1 is undefined, cannot expand struct ThisModuleWillNeverBeAvailable"
     assert capture_io(fn ->
-      assert catch_exit(Kernel.ParallelCompiler.files(fixtures)) == {:shutdown, 1}
+      assert {:error, [{^fixture, 7, msg}], []} = Kernel.ParallelCompiler.files([fixture], return_errors: true)
+      assert msg =~ expected_msg
     end) =~ "== Compilation error"
   end
 
   test "handles possible deadlocks" do
-    fixtures = [fixture_path("parallel_deadlock/foo.ex"),
-                fixture_path("parallel_deadlock/bar.ex")]
+    foo = fixture_path("parallel_deadlock/foo.ex")
+    bar = fixture_path("parallel_deadlock/bar.ex")
+    fixtures = [foo, bar]
 
     msg = capture_io(fn ->
-      assert catch_exit(Kernel.ParallelCompiler.files fixtures) == {:shutdown, 1}
+      assert {:error,
+              [{^bar, nil, "deadlocked waiting on module FooDeadlock"},
+               {^foo, nil, "deadlocked waiting on module BarDeadlock"}],
+              []
+             } = Kernel.ParallelCompiler.files(fixtures, return_errors: true)
     end)
 
     assert msg =~ "Compilation failed because of a deadlock between files."
@@ -61,13 +78,17 @@ defmodule Kernel.ParallelCompilerTest do
 
   test "warnings as errors" do
     warnings_as_errors = Code.compiler_options[:warnings_as_errors]
-    fixtures = [fixture_path("warnings_sample.ex")]
+    fixture = fixture_path("warnings_sample.ex")
 
     try do
       Code.compiler_options(warnings_as_errors: true)
 
       msg = capture_io :stderr, fn ->
-        assert catch_exit(Kernel.ParallelCompiler.files fixtures) == {:shutdown, 1}
+        assert {:error,
+                [{^fixture, 3,
+                  "this clause cannot match because a previous clause at line 2 always matches"}],
+                []
+               } = Kernel.ParallelCompiler.files([fixture], return_errors: true)
       end
 
       assert msg =~ "Compilation failed due to warnings while using the --warnings-as-errors option\n"