Fix Parallel Compilation

This patch does two things:

1. it broadens the interface for the compiler module so that
   non-first-file modules can possibly be parallelized. This is done by
   dynamically switching on `[ListOfFiles]`, which remains sequential as
   before, or `{[SeqPriority], [Parallel]}`, which divides regular files
   between higher priority ones and those that can be in parallel
2. implements this mechanism in the rebar compiler, based on the erl
   file digraph. If a file has an in-neighbour, it is depended on by
   another file. The mechanism therefore makes it so all files that have
   dependants get compiled in their strict relative sequential order
   first, and then the undepended-on files get compiled together in
   parallel.

By running:

  ./rebar3 ct --suite test/rebar_compile_SUITE.erl --case \
  recompile_when_parse_transform_inline_changes --repeat 50

the previous iteration of this would rapidly fail, and this one succeeds
every time.

src/rebar_compiler.erl

@@ -21,7 +21,10 @@
                                            out_mappings => out_mappings()}.
 -callback needed_files(digraph:graph(), [file:filename()], out_mappings(),
                        rebar_app_info:t()) ->
-    {{[file:filename()], term()}, {[file:filename()], term()}}.
+    {{[file:filename()], term()}, % ErlFirstFiles (erl_opts global priority)
+     {[file:filename()] | % [Sequential]
+      {[file:filename()], [file:filename()]}, % {Sequential, Parallel}
+      term()}}.
 -callback dependencies(file:filename(), file:dirname(), [file:dirname()]) -> [file:filename()].
 -callback compile(file:filename(), out_mappings(), rebar_dict(), list()) ->
     ok | {ok, [string()]} | {ok, [string()], [string()]}.
@@ -77,7 +80,13 @@ run(CompilerMod, AppInfo, Label) ->
     true = digraph:delete(G),
 
     compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
-    compile_parallel(RestFiles, Opts, BaseOpts, Mappings, CompilerMod).
+    case RestFiles of
+        {Sequential, Parallel} -> % new parallelizable form
+            compile_each(Sequential, Opts, BaseOpts, Mappings, CompilerMod),
+            compile_parallel(Parallel, Opts, BaseOpts, Mappings, CompilerMod);
+        _ when is_list(RestFiles) -> % traditional sequential build
+            compile_each(RestFiles, Opts, BaseOpts, Mappings, CompilerMod)
+    end.
 
 compile_each([], _Opts, _Config, _Outs, _CompilerMod) ->
     ok;

src/rebar_compiler_erl.erl

@@ -55,7 +55,14 @@ needed_files(Graph, FoundFiles, _, AppInfo) ->
     {ErlFirstFiles, ErlOptsFirst} = erl_first_files(RebarOpts, ErlOpts, Dir, NeededErlFiles),
     SubGraph = digraph_utils:subgraph(Graph, NeededErlFiles),
     DepErlsOrdered = digraph_utils:topsort(SubGraph),
-    OtherErls = lists:reverse(DepErlsOrdered),
+    %% Break out the files required by other modules from those
+    %% that none other depend of; the former must be sequentially
+    %% built, the rest is parallelizable.
+    OtherErls = lists:partition(
+        fun(Erl) -> digraph:in_degree(Graph, Erl) > 0 end,
+        lists:reverse([Dep || Dep <- DepErlsOrdered,
+                              not lists:member(Dep, ErlFirstFiles)])
+    ),
 
     PrivIncludes = [{i, filename:join(OutDir, Src)}
                     || Src <- rebar_dir:all_src_dirs(RebarOpts, ["src"], [])],
@@ -64,8 +71,7 @@ needed_files(Graph, FoundFiles, _, AppInfo) ->
     true = digraph:delete(SubGraph),
 
     {{ErlFirstFiles, ErlOptsFirst ++ AdditionalOpts},
-     {[Erl || Erl <- OtherErls,
-              not lists:member(Erl, ErlFirstFiles)], ErlOpts ++ AdditionalOpts}}.
+     {OtherErls, ErlOpts ++ AdditionalOpts}}.
 
 dependencies(Source, SourceDir, Dirs) ->
     {ok, Fd} = file:open(Source, [read]),