From 5f03505ebab0ad5160bc6794b8b5c0bf0001480a Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Wed, 29 Jan 2020 13:30:05 -0700 Subject: [PATCH] initial support for relx 4.0.0 This commit switches the relx dependency to branch 4.0.0. It is the first iteration of integration with relx 4.0 which adds many changes to how relx works and how the two integrate. rebar_relx module now creates a relx state from the relx configuration and pass it along with all the OTP applications rebar3 knows about which may be used in the release. This allows relx to not have to duplicate the effort of application discovery like before. Since this is intended to be released only when OTP-23 is released the testing of OTP-18 has been removed. --- .cirrus.yml | 2 +- rebar.config | 5 +- rebar.lock | 22 ++---- src/rebar_app_discover.erl | 10 +-- src/rebar_app_info.erl | 75 +++++++++++++------ src/rebar_prv_release.erl | 5 +- src/rebar_prv_relup.erl | 4 +- src/rebar_prv_tar.erl | 4 +- src/rebar_relx.erl | 139 +++++++++++++++++++++++------------ test/rebar_release_SUITE.erl | 6 +- test/rebar_test_utils.erl | 6 +- 11 files changed, 171 insertions(+), 107 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7b3dba0c..ba537814 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -2,11 +2,11 @@ test_task: use_compute_credits: true container: matrix: + - image: erlang:23 - image: erlang:22 - image: erlang:21 - image: erlang:20 - image: erlang:19 - - image: erlang:18 test_script: | ./bootstrap ./rebar3 ct diff --git a/rebar.config b/rebar.config index 1becefce..dc1eceb1 100644 --- a/rebar.config +++ b/rebar.config @@ -8,7 +8,8 @@ {providers, "1.8.1"}, {getopt, "1.0.1"}, {bbmustache, "1.8.0"}, - {relx, "3.33.0"}, + %% {relx, "3.33.0"}, + {relx, {git, "https://github.com/erlware/relx.git", {branch, "4.0.0"}}}, {cf, "0.2.2"}, {cth_readable, "1.4.8"}, {eunit_formatters, "0.5.0"}]}. @@ -29,6 +30,8 @@ {escript_incl_extra, [{"relx/priv/templates/*", "_build/default/lib/"}, {"rebar/priv/templates/*", "_build/default/lib/"}]}. +{overrides, [{add, relx, [{erl_opts, [{d, 'RLX_LOG', rebar_log}]}]}]}. + {erl_opts, [{platform_define, "^(19|2)", rand_only}, {platform_define, "^18", no_maps_update_with}, {platform_define, "^2", unicode_str}, diff --git a/rebar.lock b/rebar.lock index 4629a239..50697b43 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,4 +1,4 @@ -{"1.2.0", +{"1.1.0", [{<<"bbmustache">>,{pkg,<<"bbmustache">>,<<"1.8.0">>},0}, {<<"certifi">>,{pkg,<<"certifi">>,<<"2.5.1">>},0}, {<<"cf">>,{pkg,<<"cf">>,<<"0.2.2">>},0}, @@ -8,7 +8,10 @@ {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},0}, {<<"providers">>,{pkg,<<"providers">>,<<"1.8.1">>},0}, - {<<"relx">>,{pkg,<<"relx">>,<<"3.33.0">>},0}, + {<<"relx">>, + {git,"https://github.com/erlware/relx.git", + {ref,"ad25111b7349c633fbec30f26c1ad03361d6b806"}}, + 0}, {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.5">>},0}]}. [ {pkg_hash,[ @@ -21,18 +24,5 @@ {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>}, {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, {<<"providers">>, <<"70B4197869514344A8A60E2B2A4EF41CA03DEF43CFB1712ECF076A0F3C62F083">>}, - {<<"relx">>, <<"AFC019320BB69881718576B3E4E1EB548C1FA3270717BA66A78004C98A77CD17">>}, - {<<"ssl_verify_fun">>, <<"6EAF7AD16CB568BB01753DBBD7A95FF8B91C7979482B95F38443FE2C8852A79B">>}]}, -{pkg_hash_ext,[ - {<<"bbmustache">>, <<"190EA2206128BDFABF5D9200B8DF97F6511D9C62953655828E28C2BC79161252">>}, - {<<"certifi">>, <<"805ABD97539CAF89EC6D4732C91E62BA9DA0CDA51AC462380BBD28EE697A8C42">>}, - {<<"cf">>, <<"48283B3019BC7FAD56E7B23028A5DA4D3E6CD598A553AB2A99A2153BF5F19B21">>}, - {<<"cth_readable">>, <<"46C3BB14DF581DC7A9DC0CB9E8C755BFF596665FB9A23148DD76E3A200804E90">>}, - {<<"erlware_commons">>, <<"7AADA93F368D0A0430122E39931B7FB4AC9E94DBF043CDC980AD4330FD9CD166">>}, - {<<"eunit_formatters">>, <<"D6C8BA213424944E6E05BBC097C32001CDD0ABE3925D02454F229B20D68763C9">>}, - {<<"getopt">>, <<"53E1AB83B9CEB65C9672D3E7A35B8092E9BDC9B3EE80721471A161C10C59959C">>}, - {<<"parse_trans">>, <<"17EF63ABDE837AD30680EA7F857DD9E7CED9476CDD7B0394432AF4BFC241B960">>}, - {<<"providers">>, <<"E45745ADE9C476A9A469EA0840E418AB19360DC44F01A233304E118A44486BA0">>}, - {<<"relx">>, <<"6E0456139FC70BADE0C45FF8A8197C5E879A57FD792F771FC632B94C5AEC1EAC">>}, - {<<"ssl_verify_fun">>, <<"13104D7897E38ED7F044C4DE953A6C28597D1C952075EB2E328BC6D6F2BFC496">>}]} + {<<"ssl_verify_fun">>, <<"6EAF7AD16CB568BB01753DBBD7A95FF8B91C7979482B95F38443FE2C8852A79B">>}]} ]. diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 0c15bb29..ca837c54 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -362,16 +362,16 @@ create_app_info(AppInfo, AppDir, AppFile) -> rebar_app_info:original_vsn( rebar_app_info:dir(AppInfo, AppDir), AppVsn), AppName), AppInfo2 = rebar_app_info:applications( - rebar_app_info:app_details(AppInfo1, AppDetails), - IncludedApplications++Applications), - Valid = case rebar_app_utils:validate_application_info(AppInfo2) =:= true - andalso rebar_app_info:has_all_artifacts(AppInfo2) =:= true of + rebar_app_info:app_details(AppInfo1, AppDetails), Applications), + AppInfo3 = rebar_app_info:included_applications(AppInfo2, IncludedApplications), + Valid = case rebar_app_utils:validate_application_info(AppInfo3) =:= true + andalso rebar_app_info:has_all_artifacts(AppInfo3) =:= true of true -> true; _ -> false end, - rebar_app_info:dir(rebar_app_info:valid(AppInfo2, Valid), AppDir); + rebar_app_info:dir(rebar_app_info:valid(AppInfo3, Valid), AppDir); _Invalid -> throw({error, {?MODULE, {cannot_read_app_file, AppFile}}}) catch diff --git a/src/rebar_app_info.erl b/src/rebar_app_info.erl index d051a156..4b0157f8 100644 --- a/src/rebar_app_info.erl +++ b/src/rebar_app_info.erl @@ -6,6 +6,7 @@ new/3, new/4, new/5, + app_to_map/1, update_opts/3, update_opts/2, update_opts_deps/2, @@ -27,6 +28,8 @@ priv_dir/1, applications/1, applications/2, + included_applications/1, + included_applications/2, profiles/1, profiles/2, deps/1, @@ -76,28 +79,29 @@ -type project_type() :: rebar3 | mix | undefined. --record(app_info_t, {name :: binary() | undefined, - app_file_src :: file:filename_all() | undefined, - app_file_src_script:: file:filename_all() | undefined, - app_file :: file:filename_all() | undefined, - original_vsn :: binary() | undefined, - parent=root :: binary() | root, - app_details=[] :: list(), - applications=[] :: list(), - deps=[] :: list(), - profiles=[default] :: [atom()], - default=dict:new() :: rebar_dict(), - opts=dict:new() :: rebar_dict(), - dep_level=0 :: integer(), - dir :: file:name(), - out_dir :: file:name(), - ebin_dir :: file:name(), - source :: string() | tuple() | checkout | undefined, - is_lock=false :: boolean(), - is_checkout=false :: boolean(), - valid :: boolean() | undefined, - project_type :: project_type(), - is_available=false :: boolean()}). +-record(app_info_t, {name :: binary() | undefined, + app_file_src :: file:filename_all() | undefined, + app_file_src_script :: file:filename_all() | undefined, + app_file :: file:filename_all() | undefined, + original_vsn :: binary() | undefined, + parent=root :: binary() | root, + app_details=[] :: list(), + applications=[] :: list(), + included_applications=[] :: [atom()], + deps=[] :: list(), + profiles=[default] :: [atom()], + default=dict:new() :: rebar_dict(), + opts=dict:new() :: rebar_dict(), + dep_level=0 :: integer(), + dir :: file:name(), + out_dir :: file:name(), + ebin_dir :: file:name(), + source :: string() | tuple() | checkout | undefined, + is_lock=false :: boolean(), + is_checkout=false :: boolean(), + valid :: boolean() | undefined, + project_type :: project_type(), + is_available=false :: boolean()}). %%============================================================================ %% types @@ -159,6 +163,22 @@ new(Parent, AppName, Vsn, Dir, Deps) -> ebin_dir=filename:join(rebar_utils:to_list(Dir), "ebin"), deps=Deps}}. +app_to_map(#app_info_t{name=Name, + original_vsn=Vsn, + applications=Applications, + included_applications=IncludedApplications, + out_dir=OutDir, + ebin_dir=EbinDir}) -> + %% TODO: call rlx_app_info to create map + #{name => ec_cnv:to_atom(Name), + vsn => Vsn, + applications => Applications, + included_applications => IncludedApplications, + dir => OutDir, + out_dir => OutDir, + ebin_dir => EbinDir, + link => false}. + %% @doc update the opts based on the contents of a config %% file for the app -spec update_opts(t(), rebar_dict(), [any()]) -> t(). @@ -406,6 +426,17 @@ applications(#app_info_t{applications=Applications}) -> applications(AppInfo=#app_info_t{}, Applications) -> AppInfo#app_info_t{applications=Applications}. +%% @doc returns the list of included_applications the app depends on. +-spec included_applications(t()) -> list(). +included_applications(#app_info_t{included_applications=Applications}) -> + Applications. + +%% @doc sets the list of applications the app depends on. +%% Should be obtained from the app file. +-spec included_applications(t(), list()) -> t(). +included_applications(AppInfo=#app_info_t{}, Applications) -> + AppInfo#app_info_t{included_applications=Applications}. + %% @doc returns the list of active profiles -spec profiles(t()) -> list(). profiles(#app_info_t{profiles=Profiles}) -> diff --git a/src/rebar_prv_release.erl b/src/rebar_prv_release.erl index 2cf9b23e..f44a488c 100644 --- a/src/rebar_prv_release.erl +++ b/src/rebar_prv_release.erl @@ -9,6 +9,7 @@ do/1, format_error/1]). +-include_lib("providers/include/providers.hrl"). -include("rebar.hrl"). -define(PROVIDER, release). @@ -27,12 +28,12 @@ init(State) -> {example, "rebar3 release"}, {short_desc, "Build release of project."}, {desc, "Build release of project."}, - {opts, relx:opt_spec_list()}])), + {opts, rebar_relx:opt_spec_list()}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_relx:do(rlx_prv_release, "release", ?PROVIDER, State). + rebar_relx:do(?PROVIDER, State). -spec format_error(any()) -> iolist(). format_error(Reason) -> diff --git a/src/rebar_prv_relup.erl b/src/rebar_prv_relup.erl index a4cd8aed..81460419 100644 --- a/src/rebar_prv_relup.erl +++ b/src/rebar_prv_relup.erl @@ -27,13 +27,13 @@ init(State) -> {example, "rebar3 relup"}, {short_desc, "Create relup of releases."}, {desc, "Create relup of releases."}, - {opts, relx:opt_spec_list()}]), + {opts, rebar_relx:opt_spec_list()}]), State1 = rebar_state:add_provider(State, Provider), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_relx:do(rlx_prv_release, "relup", ?PROVIDER, State). + rebar_relx:do(?PROVIDER, State). -spec format_error(any()) -> iolist(). format_error(Reason) -> diff --git a/src/rebar_prv_tar.erl b/src/rebar_prv_tar.erl index b3a12c00..cdb546d4 100644 --- a/src/rebar_prv_tar.erl +++ b/src/rebar_prv_tar.erl @@ -27,12 +27,12 @@ init(State) -> {example, "rebar3 tar"}, {short_desc, "Tar archive of release built of project."}, {desc, "Tar archive of release built of project."}, - {opts, relx:opt_spec_list()}])), + {opts, rebar_relx:opt_spec_list()}])), {ok, State1}. -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. do(State) -> - rebar_relx:do(rlx_prv_release, "tar", ?PROVIDER, State). + rebar_relx:do(?PROVIDER, State). -spec format_error(any()) -> iolist(). format_error(Reason) -> diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 996c0a3a..5a906043 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -3,71 +3,76 @@ -module(rebar_relx). --export([do/4, +-export([do/2, + opt_spec_list/0, format_error/1]). -ifdef(TEST). -export([merge_overlays/1]). -endif. +-include_lib("providers/include/providers.hrl"). -include("rebar.hrl"). %% =================================================================== %% Public API %% =================================================================== --spec do(atom(), string(), atom(), rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. -do(Module, Command, Provider, State) -> - %% We set the color mode for relx as a application env - application:set_env(relx, color_intensity, rebar_log:intensity()), - LogLevel = rebar_log:get_level(), - Options = rebar_state:command_args(State), - DepsDir = rebar_dir:deps_dir(State), - ProjectAppDirs = lists:delete(".", ?DEFAULT_PROJECT_APP_DIRS), - LibDirs = rebar_utils:filtermap(fun ec_file:exists/1, - [rebar_dir:checkouts_dir(State), DepsDir | ProjectAppDirs]), - OutputDir = filename:join(rebar_dir:base_dir(State), ?DEFAULT_RELEASE_DIR), +-spec do(atom(), rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. +do(Provider, State) -> + {Opts, _} = rebar_state:command_parsed_args(State), + Release = case proplists:get_value(relname, Opts, undefined) of + undefined -> + undefined; + R -> + case proplists:get_value(relvsn, Opts, undefined) of + undefined -> + list_to_atom(R); + RelVsn -> + {list_to_atom(R), RelVsn} + end + end, + + %% TODO: read in ./relx.config if it exists or --config value + RelxConfig = rebar_state:get(State, relx, []), + ProfileString = rebar_dir:profile_dir_name(State), - AllOptions = rebar_string:join([Command | Options], " "), - Cwd = rebar_state:dir(State), - Providers = rebar_state:providers(State), - RebarOpts = rebar_state:opts(State), ExtraOverlays = [{profile_string, ProfileString}], - ErlOpts = rebar_opts:erl_opts(RebarOpts), + + DefaultOutputDir = filename:join(rebar_dir:base_dir(State), ?DEFAULT_RELEASE_DIR), + RelxConfig1 = [output_dir(DefaultOutputDir, Opts), + {overlay_vars_values, ExtraOverlays}, + {overlay_vars, [{base_dir, rebar_dir:base_dir(State)}]} + | merge_overlays(RelxConfig)], + {ok, RelxState} = rlx_config:to_state(RelxConfig1), + + + Providers = rebar_state:providers(State), + Cwd = rebar_state:dir(State), rebar_hooks:run_project_and_app_hooks(Cwd, pre, Provider, Providers, State), - try - case rebar_state:get(State, relx, []) of - [] -> - relx:main([{lib_dirs, LibDirs} - ,{caller, api} - ,{log_level, LogLevel} - ,{api_caller_overlays, ExtraOverlays} - | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions); - Config -> - Config1 = [{overlay_vars, [{base_dir, rebar_dir:base_dir(State)}]} - | merge_overlays(Config)], - relx:main([{lib_dirs, LibDirs} - ,{config, Config1} - ,{caller, api} - ,{log_level, LogLevel} - ,{api_caller_overlays, ExtraOverlays} - | output_dir(OutputDir, Options)] ++ ErlOpts, AllOptions) - end, - rebar_hooks:run_project_and_app_hooks(Cwd, post, Provider, Providers, State), - {ok, State} - catch - throw:T -> - {error, {Module, T}} - end. + + case Provider of + release -> + relx:build_release(Release, all_apps(State), RelxState); + tar -> + relx:build_tar(Release, all_apps(State), RelxState); + relup -> + ToVsn = proplists:get_value(relvsn, Opts, undefined), + UpFromVsn = proplists:get_value(upfrom, Opts, undefined), + relx:build_relup(Release, ToVsn, UpFromVsn, RelxState) + end, + + rebar_hooks:run_project_and_app_hooks(Cwd, post, Provider, Providers, State), + + {ok, State}. -spec format_error(any()) -> iolist(). -format_error(Reason) -> - io_lib:format("~p", [Reason]). +format_error(Error) -> + io_lib:format("~p", [Error]). %% Don't override output_dir if the user passed one on the command line -output_dir(OutputDir, Options) -> - [{output_dir, OutputDir} || not(lists:member("-o", Options)) - andalso not(lists:member("--output-dir", Options))]. +output_dir(DefaultOutputDir, Options) -> + {output_dir, proplists:get_value(output_dir, Options, DefaultOutputDir)}. merge_overlays(Config) -> {Overlays, Others} = @@ -77,3 +82,45 @@ merge_overlays(Config) -> %% Have profile overlay entries come before others to match how profiles work elsewhere NewOverlay = lists:flatmap(fun({overlay, Overlay}) -> Overlay end, lists:reverse(Overlays)), [{overlay, NewOverlay} | Others]. + +%% + +%% Returns a map of all apps that are part of the rebar3 project. +%% This means the project apps and dependencies but not OTP libraries. +-spec all_apps(rebar_state:t()) -> #{atom() => rlx_app_info:t()}. +all_apps(State) -> + lists:foldl(fun(AppInfo, Acc) -> + Acc#{binary_to_atom(rebar_app_info:name(AppInfo), utf8) + => rebar_app_info:app_to_map(AppInfo)} + end, #{}, rebar_state:project_apps(State) ++ rebar_state:all_deps(State)). + +%% + +-spec opt_spec_list() -> [getopt:option_spec()]. +opt_spec_list() -> + [{relname, $n, "relname", string, + "Specify the name for the release that will be generated"}, + {relvsn, $v, "relvsn", string, "Specify the version for the release"}, + {upfrom, $u, "upfrom", string, + "Only valid with relup target, specify the release to upgrade from"}, + {output_dir, $o, "output-dir", string, + "The output directory for the release. This is `./` by default."}, + {help, $h, "help", undefined, + "Print usage"}, + {lib_dir, $l, "lib-dir", string, + "Additional dir that should be searched for OTP Apps"}, + {log_level, $V, "verbose", {integer, 2}, + "Verbosity level, maybe between 0 and 3"}, + {dev_mode, $d, "dev-mode", boolean, + "Symlink the applications and configuration into the release instead of copying"}, + {include_erts, $i, "include-erts", string, + "If true include a copy of erts used to build with, if a path include erts at that path. If false, do not include erts"}, + {override, $a, "override", string, + "Provide an app name and a directory to override in the form :"}, + {config, $c, "config", {string, ""}, "The path to a config file"}, + {overlay_vars, undefined, "overlay_vars", string, "Path to a file of overlay variables"}, + {vm_args, undefined, "vm_args", string, "Path to a file to use for vm.args"}, + {sys_config, undefined, "sys_config", string, "Path to a file to use for sys.config"}, + {system_libs, undefined, "system_libs", string, "Boolean or path to dir of Erlang system libs"}, + {version, undefined, "version", undefined, "Print relx version"}, + {root_dir, $r, "root", string, "The project root directory"}]. diff --git a/test/rebar_release_SUITE.erl b/test/rebar_release_SUITE.erl index 1785a972..d264cb8b 100644 --- a/test/rebar_release_SUITE.erl +++ b/test/rebar_release_SUITE.erl @@ -190,11 +190,7 @@ user_output_dir(Config) -> {ok, []} ), - RelxState = rlx_state:new("", [], []), - RelxState1 = rlx_state:base_output_dir(RelxState, ReleaseDir), - {ok, RelxState2} = rlx_prv_app_discover:do(RelxState1), - {ok, RelxState3} = rlx_prv_rel_discover:do(RelxState2), - rlx_state:get_realized_release(RelxState3, list_to_atom(Name), Vsn). + ?assertNotMatch([], filelib:wildcard(filename:join([ReleaseDir, Name, "releases", Vsn, "*"]))). profile_overlays(Config) -> AppDir = ?config(apps, Config), diff --git a/test/rebar_test_utils.erl b/test/rebar_test_utils.erl index bc5d35d7..0f5948d2 100644 --- a/test/rebar_test_utils.erl +++ b/test/rebar_test_utils.erl @@ -392,10 +392,6 @@ check_results(AppDir, Expected, ProfileRun) -> try file:set_cwd(AppDir), [ReleaseDir] = filelib:wildcard(filename:join([AppDir, "_build", "*", "rel"])), - RelxState = rlx_state:new("", [], []), - RelxState1 = rlx_state:base_output_dir(RelxState, ReleaseDir), - {ok, RelxState2} = rlx_prv_app_discover:do(RelxState1), - {ok, RelxState3} = rlx_prv_rel_discover:do(RelxState2), LibDir = filename:join([ReleaseDir, Name, "lib"]), {ok, RelLibs} = rebar_utils:list_dir(LibDir), @@ -407,7 +403,7 @@ check_results(AppDir, Expected, ProfileRun) -> ?assertEqual(ExpectedDevMode, DevMode), %% throws not_found if it doesn't exist - rlx_state:get_realized_release(RelxState3, Name, Vsn) + ok catch _ -> ct:fail(release_not_found)