diff --git a/.gitignore b/.gitignore index 751a61d..0b66f73 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ deps .rebar3 _build/ _checkouts/ + +.idea + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1a0f2c9 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +all: + ./bootstrap.erl + +debug: + ./bootstrap debug + +xref: debug + ./xref + +clean: + @rm -rf rebar .rebar ebin/*.beam diff --git a/README.md b/README.md new file mode 100644 index 0000000..68c715c --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# Erlang nif port compiler + windows linux mac下erlang nif或者port_driver通用编译脚本 + 改造自 erlang-native-compiler + + +## Usage +default_env +1. Clone this repository +1. Run `make` in this directory +1. Copy `erlNpc` to your project "c_src" dir and commit it +1. Add these (or similar) hooks to your rebar.config: + +```erlang +{pre_hooks, [{"", compile, "escript c_src/erlNpc compile"}]}. +{post_hooks, [{"", clean, "escript c_src/erlNpc clean"}]}. +``` + +After that enc should read your old rebar.config `port\_specs` and `port\_env` settings as expected (it is rebar2's port compiler after all...). \ No newline at end of file diff --git a/bootstrap b/bootstrap new file mode 100644 index 0000000..6e7ff3c --- /dev/null +++ b/bootstrap @@ -0,0 +1,212 @@ +#!/usr/bin/env escript +%% -*- erlang -* +%%! + +-include_lib("kernel/include/file.hrl"). + +main(Args) -> + case lists:member("--help", Args) of + true -> + usage(), + halt(0); + false -> + ok + end, + + %% Get a string repr of build time + BuiltTime = getTimeStr(), + + %% Get a string repr of first matching VCS changeset + VcsInfo = vcsInfo([{git, ".git", "git describe --always --tags", "git status -s"}]), + + %% Check for force=1 flag to force a rebuild + case lists:member("force=1", Args) of + true -> + rm("ebin/*.beam"); + false -> + case filelib:is_file("ebin/rebar.beam") of + true -> rm("ebin/rebar.beam"); + false -> io:fwrite("No beam files found.~n") + end + end, + + %% Add check for debug flag + DebugFlag = + case lists:member("debug", Args) of + true -> debug_info; + false -> undefined + end, + + OtpInfo = string:strip(erlang:system_info(otp_release), both, $\n), + + %% Types dict:dict() and digraph:digraph() have been introduced in + %% Erlang 17. + %% At the same time, their counterparts dict() and digraph() are to be + %% deprecated in Erlang 18. namespaced_types option is used to select + %% proper type name depending on the OTP version used. + %% Extract the system info of the version of OTP we use to compile rebar + %% NamespacedTypes = + %% case is_otp(OtpInfo, "^[0-9]+") of + %% true -> {d, namespaced_types}; + %% false -> undefined + %% end, + + %% Compile all src/*.erl to ebin + %% To not accidentally try to compile files like Mac OS X resource forks, + %% we only look for rebar source files that start with a letter. + Opts = [ + DebugFlag, + {outdir, "ebin"}, + {i, "include"}, + {d, 'BUILD_TIME', BuiltTime}, + {d, 'VCS_INFO', VcsInfo}, + {d, 'OTP_INFO', OtpInfo} + ], + case make:files(filelib:wildcard("src/*.erl"), Opts) of + up_to_date -> + ok; + error -> + io:format("Failed to compile erlNpc files!\n"), + halt(1) + end, + + %% Make sure file:consult can parse the .app file + case file:consult("ebin/erlNpc.app") of + {ok, _} -> + ok; + {error, Reason} -> + io:format("Invalid syntax in ebin/erlNpc.app: ~p\n", [Reason]), + halt(1) + end, + + %% Add ebin/ to our path + true = code:add_path("ebin"), + + %% Run rebar compile to do proper .app validation etc. + %% and rebar escriptize to create the rebar script + %% RebarArgs = Args -- ["debug"], %% Avoid trying to run 'debug' command + % rebar:main(["compile"] ++ RebarArgs), + + escriptize(), + + %% Finally, update executable perms for our script on *nix, + %% or write out script files on win32. + case os:type() of + {unix, _} -> + [] = os:cmd("chmod u+x erlNpc"), + ok; + {win32, _} -> + writeWindowsScripts(), + ok; + _ -> + ok + end, + + %% Add a helpful message + io:format(<<"Congratulations! You now have a self-contained script called" + " \"erlNpc\" in\n" + "your current working directory. " + "Place this script anywhere in your path\n" + "and you can use erlNpc to build native code for Erlang\n">>). + +usage() -> + io:format(<<"Usage: bootstrap [OPTION]...~n">>), + io:format(<<" force=1 unconditional build~n">>), + io:format(<<" debug add debug information~n">>). + +%% is_otp(OtpInfo, Regex) -> +%% case re:run(OtpInfo, Regex, [{capture, none}]) of +%% match -> true; +%% nomatch -> false +%% end. + +rm(Path) -> + NativePath = filename:nativename(Path), + Cmd = case os:type() of + {unix, _} -> "rm -f "; + {win32, _} -> "del /q " + end, + [] = os:cmd(Cmd ++ NativePath), + ok. + +getTimeStr() -> + {{Y, M, D}, {H, Min, S}} = erlang:localtime(), + lists:flatten(io_lib:format("~B_~2.10.0B_~2.10.0B ~B:~2.10.0B:~2.10.0B", [Y, M, D, H, Min, S])). + +vcsInfo([]) -> + "No VCS info available."; +vcsInfo([{Id, Dir, VsnCmd, StatusCmd} | Rest]) -> + case filelib:is_dir(Dir) of + true -> + Vsn = string:strip(os:cmd(VsnCmd), both, $\n), + Status = + case string:strip(os:cmd(StatusCmd), both, $\n) of + [] -> + ""; + _ -> + "-dirty" + end, + lists:concat([Id, " ", Vsn, Status]); + false -> + vcsInfo(Rest) + end. + +writeWindowsScripts() -> + CmdScript = + "@echo off\r\n" + "setlocal\r\n" + "set rebarscript=%~f0\r\n" + "escript.exe \"%rebarscript:.cmd=%\" %*\r\n", + ok = file:write_file("erlNpc.cmd", CmdScript). + + +escriptize() -> + AppName = "erlNpc", + ScriptName = "erlNpc", + + Files = loadEScriptFiles(AppName, "ebin", "*"), + + {ok, {"mem", ZipBin}} = zip:create("mem", Files, [memory]), + Shebang = "#!/usr/bin/env escript\n", + Comment = "%%\n", + EmuArgs = io_lib:format("%%! -pa ~s/~s/ebin\n", [AppName, AppName]), + Script = iolist_to_binary([Shebang, Comment, EmuArgs, ZipBin]), + ok = file:write_file(ScriptName, Script), + + %% Finally, update executable perms for our script + {ok, #file_info{mode = Mode}} = file:read_file_info(ScriptName), + ok = file:change_mode(ScriptName, Mode bor 8#00111). + +loadEScriptFiles(AppName, Path, WildCard) -> + FileNames = filelib:wildcard(WildCard, Path), + Entries = + lists:flatmap( + fun(FN) -> + FPath = filename:join(Path, FN), + {ok, Contents} = file:read_file(FPath), + ZipPath = filename:join(AppName, FPath), + [{ZipPath, Contents} | dirEntries(ZipPath)] + end, + FileNames), + usort(Entries). + +%% Given a filename, return zip archive dir entries for each sub-dir. +%% Required to work around issues fixed in OTP-10071. +dirEntries(File) -> + Dirs = dirs(File), + [{Dir ++ "/", <<>>} || Dir <- Dirs]. + +%% Given "foo/bar/baz", return ["foo", "foo/bar", "foo/bar/baz"]. +dirs(Dir) -> + dirs1(filename:split(Dir), "", []). + +dirs1([], _, Acc) -> + lists:reverse(Acc); +dirs1([H | T], "", []) -> + dirs1(T, H, [H]); +dirs1([H | T], Last, Acc) -> + Dir = filename:join(Last, H), + dirs1(T, Dir, [Dir | Acc]). + +usort(List) -> + lists:ukeysort(1, lists:flatten(List)). diff --git a/bootstrap.bat b/bootstrap.bat new file mode 100644 index 0000000..b646a7d --- /dev/null +++ b/bootstrap.bat @@ -0,0 +1,2 @@ +@echo off +escript.exe bootstrap %* diff --git a/ebin/erlNpc.app b/ebin/erlNpc.app new file mode 100644 index 0000000..76ed1ee --- /dev/null +++ b/ebin/erlNpc.app @@ -0,0 +1,9 @@ +{application,erlNpc, + [{description,"erlNpc: Erlang Native Compiler"}, + {vsn,"0.1.0"}, + {registered,[]}, + {applications,[kernel,stdlib]}, + {modules,[erlNpc,rebar,rebarConfig,rebarNpCompiler,rebarUtils]}, + {licenses,["Apache 2.0"]}, + {links,[]}, + {env,[{log_level,warn}]}]}. diff --git a/include/rebar.hrl b/include/rebar.hrl new file mode 100644 index 0000000..1b30a11 --- /dev/null +++ b/include/rebar.hrl @@ -0,0 +1,9 @@ +-define(FAIL, rebarUtils:abort()). +-define(ABORT(Str, Args), rebarUtils:abort(Str, Args)). + +-define(INFO(Str, Args), rebar:log(info, Str, Args)). +-define(WARN(Str, Args), rebar:log(warn, Str, Args)). +-define(ERROR(Str, Args), rebar:log(error, Str, Args)). + +-define(CONSOLE(Str, Args), io:format(Str, Args)). +-define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))). diff --git a/rebar.config b/rebar.config new file mode 100644 index 0000000..f618f3e --- /dev/null +++ b/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. \ No newline at end of file diff --git a/src/erlNpc.erl b/src/erlNpc.erl new file mode 100644 index 0000000..64d6387 --- /dev/null +++ b/src/erlNpc.erl @@ -0,0 +1,31 @@ +-module(erlNpc). + +-export([ + main/1 +]). + +main(Args) -> + file:set_cwd("c_src"), + {ok, Dir} = file:get_cwd(), + io:format("erlNpc begin compile pwd:~p ~n", [Dir]), + FunCom = + fun(File) -> + case filelib:is_dir(File) == true andalso lists:nth(1, File) =/= 46 andalso filename:basename(File) =/= "include" of + true -> + {ok, CurDir} = file:get_cwd(), + io:format("erlNpc cur compile/clean: ~s, cur pwd:~p ~n", [File, CurDir]), + file:set_cwd(File), + rebar:main(Args), + file:set_cwd(".."); + _ -> + ignore + end + end, + case file:list_dir(".") of + {ok, Files} -> + lists:foreach(FunCom, Files); + _Err -> + rebar:log(error, "erlNpc start compile error ~p ~n", [_Err]) + + end. + diff --git a/src/rebar.erl b/src/rebar.erl new file mode 100644 index 0000000..ee71287 --- /dev/null +++ b/src/rebar.erl @@ -0,0 +1,230 @@ +-module(rebar). + +-export([ + main/1, + log/3 +]). + +-include("rebar.hrl"). + +-ifndef(BUILD_TIME). +-define(BUILD_TIME, "undefined"). +-endif. + +-ifndef(VCS_INFO). +-define(VCS_INFO, "undefined"). +-endif. + +-ifndef(OTP_INFO). +-define(OTP_INFO, "undefined"). +-endif. + +-define(DEFAULT_JOBS, 3). + +main(Args) -> + case catch (run(Args)) of + ok -> + ok; + rebar_abort -> + rebarUtils:delayedHalt(1); + Error -> + %% Nothing should percolate up from rebar_core; + %% Dump this error to console + io:format("Uncaught error in rebar_core: ~p\n", [Error]), + rebarUtils:delayedHalt(1) + end. + +log(Level, Format, Args) -> + {ok, LimitLevel} = application:get_env(erlNpc, log_level), + case levelInt(LimitLevel) >= levelInt(Level) of + true -> + io:format(destination(Level), Format, Args); + false -> + ok + end. + +levelInt(info) -> 2; +levelInt(warn) -> 1; +levelInt(error) -> 0. + +destination(error) -> standard_error; +destination(_) -> group_leader(). + +run(["help"]) -> + usage(), + help(compile); +run(["help" | RawCommands]) -> + lists:foreach(fun help/1, [list_to_atom(C) || C <- RawCommands]); +run(["version"]) -> + ok = loadRebarApp(), + %% Display vsn and build time info + version(); +run(RawArgs) -> + ok = loadRebarApp(), + Args = parseArgs(RawArgs), + BaseConfig = initConfig(Args), + {BaseConfig1, Cmds} = saveOptions(BaseConfig, Args), + runAux(BaseConfig1, Cmds). + +loadRebarApp() -> + %% Pre-load the rebar app so that we get default configuration + case application:load(erlNpc) of + ok -> + ok; + {error, {already_loaded,erlNpc}} -> + ok; + _ -> + rebarUtils:delayedHalt(1) + end. + +help(compile) -> + rebarNpCompiler:info(help, compile); +help(clean) -> + rebarNpCompiler:info(help, clean); +help(Command) -> + ?CONSOLE("erlNpc no help available for \"~p\"~n", [Command]). + +parseArgs([]) -> + {[], []}; +parseArgs(["-h" | _]) -> + usage(), + help(compile), + rebarUtils:delayedHalt(0); +parseArgs(["--help" | _]) -> + usage(), + help(compile), + rebarUtils:delayedHalt(0); +parseArgs(["-v" | _]) -> + version(), + rebarUtils:delayedHalt(0); +parseArgs(["--version" | _]) -> + version(), + rebarUtils:delayedHalt(0); +parseArgs(["-c", FileName | Rest]) -> + {Opts, NonOpts} = parseArgs(Rest), + {[{config, FileName} | Opts], NonOpts}; +parseArgs(["--config", FileName | Rest]) -> + parseArgs(["-c", FileName | Rest]); +parseArgs([NonOpt | Rest]) -> + {Opts, NonOpts} = parseArgs(Rest), + {Opts, [NonOpt | NonOpts]}. + +usage() -> + ?CONSOLE("erlNpc [-hv] [-c CONFIG_FILE] COMMAND [COMMAND ...]~n~n", []). + +initConfig({Options, _NonOptArgs}) -> + %% If $HOME/.rebar/config exists load and use as global config + GlobalConfigFile = filename:join([os:getenv("HOME"), ".rebar", "config"]), + GlobalConfig = + case filelib:is_regular(GlobalConfigFile) of + true -> + rebarConfig:new(GlobalConfigFile); + false -> + rebarConfig:new() + end, + + %% Set the rebar config to use + GlobalConfig1 = + case proplists:get_value(config, Options) of + undefined -> + GlobalConfig; + Conf -> + rebarConfig:setGlobal(GlobalConfig, config, Conf) + end, + + BaseConfig = rebarConfig:baseConfig(GlobalConfig1), + + %% Keep track of how many operations we do, so we can detect bad commands + BaseConfig1 = rebarConfig:setXconf(BaseConfig, operations, 0), + %% Initialize vsn cache + rebarUtils:initVsnCache(BaseConfig1). + +initConfig_1(BaseConfig) -> + %% Determine the location of the rebar executable; important for pulling + %% resources out of the escript + ScriptName = filename:absname(escript:script_name()), + BaseConfig1 = rebarConfig:setXconf(BaseConfig, escript, ScriptName), + %% Note the top-level directory for reference + AbsCwd = filename:absname(rebarUtils:getCwd()), + rebarConfig:setXconf(BaseConfig1, base_dir, AbsCwd). + +runAux(BaseConfig, Commands) -> + %% Make sure crypto is running + case crypto:start() of + ok -> ok; + {error, {already_started, crypto}} -> ok + end, + + %% Convert command strings to atoms + CommandAtoms = [list_to_atom(C) || C <- Commands], + + BaseConfig1 = initConfig_1(BaseConfig), + + %% Make sure we're an app directory + AppFile = "", + %% case rebarUtils:isAppDir() of + %% {true, AppFile0} -> + %% AppFile0; + %% false -> + %% rebarUtils:delayedHalt(1) + %% end, + + % Setup our environment + BaseConfig2 = setupEnvs(BaseConfig1, [rebarNpCompiler]), + %% Process each command, resetting any state between each one + lists:foreach( + fun(Command) -> + %processCommand(Command, BaseConfig2, AppFile)"" + processCommand(Command, BaseConfig2, AppFile) + end, CommandAtoms). + +setupEnvs(Config, Modules) -> + lists:foldl( + fun(Module, CfgAcc) -> + Env = Module:setupEnv(CfgAcc), + rebarConfig:saveEnv(CfgAcc, Module, Env) + end, Config, Modules). + +processCommand(compile, Config, AppFile) -> + rebarNpCompiler:compile(Config, AppFile); + +processCommand(clean, Config, AppFile) -> + rebarNpCompiler:clean(Config, AppFile); + +processCommand(Other, _, _) -> + ?CONSOLE("Unknown command: ~s~n", [Other]), + rebarUtils:delayedHalt(1). + +saveOptions(Config, {Options, NonOptArgs}) -> + GlobalDefines = proplists:get_all_values(defines, Options), + Config1 = rebarConfig:setXconf(Config, defines, GlobalDefines), + filterFlags(Config1, NonOptArgs, []). + +%% show version information and halt +version() -> + {ok, Vsn} = application:get_key(erlNpc, vsn), + ?CONSOLE("erlNpc ~s ~s ~s ~s\n", [Vsn, ?OTP_INFO, ?BUILD_TIME, ?VCS_INFO]). + +%% Seperate all commands (single-words) from flags (key=value) and store +%% values into the rebar_config global storage. +filterFlags(Config, [], Commands) -> + {Config, lists:reverse(Commands)}; +filterFlags(Config, [Item | Rest], Commands) -> + case string:tokens(Item, "=") of + [Command] -> + filterFlags(Config, Rest, [Command | Commands]); + [KeyStr, RawValue] -> + Key = list_to_atom(KeyStr), + Value = + case Key of + verbose -> + list_to_integer(RawValue); + _ -> + RawValue + end, + Config1 = rebarConfig:setGlobal(Config, Key, Value), + filterFlags(Config1, Rest, Commands); + Other -> + ?CONSOLE("Ignoring command line argument: ~p\n", [Other]), + filterFlags(Config, Rest, Commands) + end. diff --git a/src/rebarConfig.erl b/src/rebarConfig.erl new file mode 100644 index 0000000..9512880 --- /dev/null +++ b/src/rebarConfig.erl @@ -0,0 +1,211 @@ +-module(rebarConfig). + +-include("rebar.hrl"). + +-export([ + new/0, + new/1, + baseConfig/1, + consultFile/1, + get/3, + getLocal/3, + getList/3, + setGlobal/3, + getGlobal/3, + saveEnv/3, + getEnv/2, + setXconf/3, + getXconf/2, + getXconf/3 +]). + +-type key() :: atom(). +-type rebarDict() :: dict:dict(term(), term()). + +-record(config, { + dir :: file:filename(), + opts = [] :: list(), + globals = newGlobals() :: rebarDict(), + envs = newEnv() :: rebarDict(), + %% cross-directory/-command config + skipDirs = newSkipDirs() :: rebarDict(), + xconf = newXconf() :: rebarDict() +}). + +-opaque config() :: #config{}. +-export_type([config/0]). + +-define(DEFAULT_NAME, "rebar.config"). + +-spec baseConfig(config()) -> config(). +baseConfig(GlobalConfig) -> + ConfName = rebarConfig:getGlobal(GlobalConfig, config, ?DEFAULT_NAME), + new(GlobalConfig, ConfName). + +-spec new() -> config(). +new() -> + #config{dir = rebarUtils:getCwd()}. + +-spec new(file:filename() | config()) -> config(). +new(ConfigFile) when is_list(ConfigFile) -> + case consultFile(ConfigFile) of + {ok, Opts} -> + #config{dir = rebarUtils:getCwd(), + opts = Opts}; + Other -> + ?ABORT("Failed to load ~s: ~p~n", [ConfigFile, Other]) + end; +new(#config{opts = Opts0, globals = Globals, skipDirs = SkipDirs, xconf = Xconf}) -> + new(#config{opts = Opts0, globals = Globals, skipDirs = SkipDirs, xconf = Xconf}, + ?DEFAULT_NAME). + +-spec get(config(), key(), term()) -> term(). +get(Config, Key, Default) -> + proplists:get_value(Key, Config#config.opts, Default). + +-spec getList(config(), key(), term()) -> term(). +getList(Config, Key, Default) -> + get(Config, Key, Default). + +-spec getLocal(config(), key(), term()) -> term(). +getLocal(Config, Key, Default) -> + proplists:get_value(Key, localOpts(Config#config.opts, []), Default). + +-spec setGlobal(config(), key(), term()) -> config(). +setGlobal(Config, jobs = Key, Value) when is_list(Value) -> + setGlobal(Config, Key, list_to_integer(Value)); +setGlobal(Config, jobs = Key, Value) when is_integer(Value) -> + NewGlobals = dict:store(Key, erlang:max(1, Value), Config#config.globals), + Config#config{globals = NewGlobals}; +setGlobal(Config, Key, Value) -> + NewGlobals = dict:store(Key, Value, Config#config.globals), + Config#config{globals = NewGlobals}. + +-spec getGlobal(config(), key(), term()) -> term(). +getGlobal(Config, Key, Default) -> + case dict:find(Key, Config#config.globals) of + error -> + Default; + {ok, Value} -> + Value + end. + +-spec consultFile(file:filename()) -> term(). +consultFile(File) -> + case filename:extension(File) of + ".script" -> + consultAndEval(removeScriptExt(File), File); + _ -> + Script = File ++ ".script", + case filelib:is_regular(Script) of + true -> + consultAndEval(File, Script); + false -> + file:consult(File) + end + end. + +-spec saveEnv(config(), module(), nonempty_list()) -> config(). +saveEnv(Config, Mod, Env) -> + NewEnvs = dict:store(Mod, Env, Config#config.envs), + Config#config{envs = NewEnvs}. + +-spec getEnv(config(), module()) -> term(). +getEnv(Config, Mod) -> + dict:fetch(Mod, Config#config.envs). + +-spec setXconf(config(), term(), term()) -> config(). +setXconf(Config, Key, Value) -> + NewXconf = dict:store(Key, Value, Config#config.xconf), + Config#config{xconf = NewXconf}. + +-spec getXconf(config(), term()) -> term(). +getXconf(Config, Key) -> + {ok, Value} = dict:find(Key, Config#config.xconf), + Value. + +-spec getXconf(config(), term(), term()) -> term(). +getXconf(Config, Key, Default) -> + case dict:find(Key, Config#config.xconf) of + error -> + Default; + {ok, Value} -> + Value + end. + +%% =================================================================== +%% Internal functions +%% =================================================================== + +-spec new(config(), file:filename()) -> config(). +new(ParentConfig, ConfName) -> + %% Load terms from rebar.config, if it exists + Dir = rebarUtils:getCwd(), + ConfigFile = filename:join([Dir, ConfName]), + Opts0 = ParentConfig#config.opts, + Opts = case consultFile(ConfigFile) of + {ok, Terms} -> + %% Found a config file with some terms. We need to + %% be able to distinguish between local definitions + %% (i.e. from the file in the cwd) and inherited + %% definitions. To accomplish this, we use a marker + %% in the proplist (since order matters) between + %% the new and old defs. + Terms ++ [local] ++ + [Opt || Opt <- Opts0, Opt /= local]; + {error, enoent} -> + [local] ++ + [Opt || Opt <- Opts0, Opt /= local]; + Other -> + ?ABORT("Failed to load ~s: ~p\n", [ConfigFile, Other]) + end, + + ParentConfig#config{dir = Dir, opts = Opts}. + +-spec consultAndEval(file:filename(), file:filename()) -> {ok, term()}. +consultAndEval(File, Script) -> + ConfigData = tryConsult(File), + file:script(Script, bs([{'CONFIG', ConfigData}, {'SCRIPT', Script}])). + +-spec removeScriptExt(file:filename()) -> file:filename(). +removeScriptExt(F) -> + "tpircs." ++ Rev = lists:reverse(F), + lists:reverse(Rev). + +-spec tryConsult(file:filename()) -> term(). +tryConsult(File) -> + case file:consult(File) of + {ok, Terms} -> + Terms; + {error, enoent} -> + []; + {error, Reason} -> + ?ABORT("Failed to read config file ~s: ~p~n", [File, Reason]) + end. + +-type bs_vars() :: [{term(), term()}]. +-spec bs(bs_vars()) -> bs_vars(). +bs(Vars) -> + lists:foldl(fun({K, V}, Bs) -> + erl_eval:add_binding(K, V, Bs) + end, erl_eval:new_bindings(), Vars). + +-spec localOpts(list(), list()) -> list(). +localOpts([], Acc) -> + lists:reverse(Acc); +localOpts([local | _Rest], Acc) -> + lists:reverse(Acc); +localOpts([Item | Rest], Acc) -> + localOpts(Rest, [Item | Acc]). + +-spec newGlobals() -> rebarDict(). +newGlobals() -> dict:new(). + +-spec newEnv() -> rebarDict(). +newEnv() -> dict:new(). + +-spec newSkipDirs() -> rebarDict(). +newSkipDirs() -> dict:new(). + +-spec newXconf() -> rebarDict(). +newXconf() -> dict:new(). diff --git a/src/rebarNpCompiler.erl b/src/rebarNpCompiler.erl new file mode 100644 index 0000000..42848fd --- /dev/null +++ b/src/rebarNpCompiler.erl @@ -0,0 +1,740 @@ +-module(rebarNpCompiler). + +-include("rebar.hrl"). + +-export([ + compile/2, + clean/2 + +]). + +%% for internal use only +-export([ + setupEnv/1, + info/2 +]). + +-record(spec, { + type :: 'drv' | 'exe', + link_lang :: 'cc' | 'cxx', + target :: file:filename(), + sources = [] :: [file:filename(), ...], + objects = [] :: [file:filename(), ...], + opts = [] :: list() | [] +}). + +compile(Config, AppFile) -> + case getSpecs(Config, AppFile) of + [] -> + ok; + Specs -> + SharedEnv = rebarConfig:getEnv(Config, ?MODULE), + + %% Compile each of the sources + NewBins = compileSources(Config, Specs, SharedEnv), + + %% Make sure that the target directories exist + + ?INFO("Using specs ~p\n", [Specs]), + lists:foreach( + fun(#spec{target = Target}) -> + ok = filelib:ensure_dir(Target) + end, Specs), + + %% Only relink if necessary, given the Target + %% and list of new binaries + lists:foreach( + fun(#spec{target = Target, objects = Bins, opts = Opts, link_lang = LinkLang}) -> + AllBins = [sets:from_list(Bins), sets:from_list(NewBins)], + Intersection = sets:intersection(AllBins), + case needsLink(Target, sets:to_list(Intersection)) of + true -> + LinkTemplate = selectLinkTemplate(LinkLang, Target), + Env = proplists:get_value(env, Opts, SharedEnv), + Cmd = expandCommand(LinkTemplate, Env, string:join(Bins, " "), Target), + rebarUtils:sh(Cmd, [{env, Env}]); + false -> + ?INFO("Skipping relink of ~s\n", [Target]), + ok + end + end, Specs) + end. + +clean(Config, AppFile) -> + case getSpecs(Config, AppFile) of + [] -> + ok; + Specs -> + lists:foreach( + fun(#spec{target = Target, objects = Objects}) -> + rebarUtils:deleteEach([Target]), + rebarUtils:deleteEach(Objects), + rebarUtils:deleteEach(portDeps(Objects)) + end, Specs) + end, + ok. + +setupEnv(Config) -> + setupEnv(Config, defaultEnv(Config)). + +%% =================================================================== +%% Internal functions +%% =================================================================== + +info(help, compile) -> + infoHelp("Build port sources"); +info(help, clean) -> + infoHelp("Delete port build results"). + +infoHelp(Description) -> + ?CONSOLE( + "~s.~n" + "~n" + "Valid rebar.config options:~n" + "port_specs - Erlang list of tuples of the forms~n" + " {ArchRegex, TargetFile, Sources, Options}~n" + " {ArchRegex, TargetFile, Sources}~n" + " {TargetFile, Sources}~n" + "~n" + " Examples:~n" + " ~p~n" + "~n" + "port_env - Erlang list of key/value pairs which will control~n" + " the environment when running the compiler and linker.~n" + " Variables set in the surrounding system shell are taken~n" + " into consideration when expanding port_env.~n" + "~n" + " By default, the following variables are defined:~n" + " CC - C compiler~n" + " CXX - C++ compiler~n" + " CFLAGS - C compiler~n" + " CXXFLAGS - C++ compiler~n" + " LDFLAGS - Link flags~n" + " ERL_CFLAGS - default -I paths for erts and ei~n" + " ERL_LDFLAGS - default -L and -lerl_interface -lei~n" + " DRV_CFLAGS - flags that will be used for compiling~n" + " DRV_LDFLAGS - flags that will be used for linking~n" + " EXE_CFLAGS - flags that will be used for compiling~n" + " EXE_LDFLAGS - flags that will be used for linking~n" + " ERL_EI_LIBDIR - ei library directory~n" + " DRV_CXX_TEMPLATE - C++ command template~n" + " DRV_CC_TEMPLATE - C command template~n" + " DRV_LINK_TEMPLATE - C Linker command template~n" + " DRV_LINK_CXX_TEMPLATE - C++ Linker command template~n" + " EXE_CXX_TEMPLATE - C++ command template~n" + " EXE_CC_TEMPLATE - C command template~n" + " EXE_LINK_TEMPLATE - C Linker command template~n" + " EXE_LINK_CXX_TEMPLATE - C++ Linker command template~n" + "~n" + " Note that if you wish to extend (vs. replace) these variables,~n" + " you MUST include a shell-style reference in your definition.~n" + " e.g. to extend CFLAGS, do something like:~n" + "~n" + " {port_env, [{\"CFLAGS\", \"$CFLAGS -MyOtherOptions\"}]}~n" + "~n" + " It is also possible to specify platform specific options~n" + " by specifying a triplet where the first string is a regex~n" + " that is checked against Erlang's system architecture string.~n" + " e.g. to specify a CFLAG that only applies to x86_64 on linux~n" + " do:~n" + " {port_env, [{\"x86_64.*-linux\", \"CFLAGS\",~n" + " \"$CFLAGS -X86Options\"}]}~n" + "~n" + "Cross-arch environment variables to configure toolchain:~n" + " REBAR_TARGET_ARCH to set the tool chain name to use~n" + " REBAR_TARGET_ARCH_WORDSIZE (optional - " + "if CC fails to determine word size)~n" + " fallback word size is 32~n" + " REBAR_TARGET_ARCH_VSN (optional - " + "if a special version of CC/CXX is requested)~n", + [ + Description, + {port_specs, [{"priv/so_name.so", ["c_src/*.c"]}, + {"linux", "priv/hello_linux", ["c_src/hello_linux.c"]}, + {"linux", "priv/hello_linux", ["c_src/*.c"], [{env, []}]}]} + ]). + +%% set REBAR_DEPS_DIR and ERL_LIBS environment variables +defaultEnv(Config) -> + BaseDir = rebarUtils:baseDir(Config), + DepsDir0 = rebarConfig:getXconf(Config, deps_dir, "deps"), + DepsDir = filename:dirname(filename:join([BaseDir, DepsDir0, "dummy"])), + + %% include rebar's DepsDir in ERL_LIBS + Separator = case os:type() of + {win32, nt} -> ";"; + _ -> ":" + end, + + ERL_LIBS = case os:getenv("ERL_LIBS") of + false -> + {"ERL_LIBS", DepsDir}; + PrevValue -> + {"ERL_LIBS", DepsDir ++ Separator ++ PrevValue} + end, + [ + {"REBAR_DEPS_DIR", DepsDir}, + ERL_LIBS + ]. + +setupEnv(Config, ExtraEnv) -> + %% Extract environment values from the config (if specified) and + %% merge with the default for this operating system. This enables + %% max flexibility for users. + DefaultEnv = filterEnv(defaultEnv(), []), + + %% Get any port-specific envs; use port_env first and then fallback + %% to port_envs for compatibility + RawPortEnv = rebarConfig:getList( + Config, + port_env, + rebarConfig:getList(Config, port_envs, [])), + + PortEnv = filterEnv(RawPortEnv, []), + Defines = getDefines(Config), + OverrideEnv = Defines ++ PortEnv ++ filterEnv(ExtraEnv, []), + RawEnv = applyDefaults(osEnv(), DefaultEnv) ++ OverrideEnv, + expandVarsLoop(mergeEachVar(RawEnv, [])). + +getDefines(Config) -> + RawDefines = rebarConfig:getXconf(Config, defines, []), + Defines = string:join(["-D" ++ D || D <- RawDefines], " "), + [{"ERL_CFLAGS", "$ERL_CFLAGS " ++ Defines}]. + +replaceExtension(File, NewExt) -> + OldExt = filename:extension(File), + replaceExtension(File, OldExt, NewExt). + +replaceExtension(File, OldExt, NewExt) -> + filename:rootname(File, OldExt) ++ NewExt. + +%% +%% == compile and link == +%% + +compileSources(Config, Specs, SharedEnv) -> + {NewBins, Db} = + lists:foldl( + fun(#spec{sources = Sources, type = Type, opts = Opts}, Acc) -> + Env = proplists:get_value(env, Opts, SharedEnv), + compileEach(Config, Sources, Type, Env, Acc) + end, {[], []}, Specs), + %% Rewrite clang compile commands database file only if something + %% was compiled. + case NewBins of + [] -> + ok; + _ -> + {ok, ClangDbFile} = file:open("compile_commands.json", [write]), + ok = io:fwrite(ClangDbFile, "[~n", []), + lists:foreach(fun(E) -> ok = io:fwrite(ClangDbFile, E, []) end, Db), + ok = io:fwrite(ClangDbFile, "]~n", []), + ok = file:close(ClangDbFile) + end, + NewBins. + +compileEach(_Config, [], _Type, _Env, {NewBins, CDB}) -> + {lists:reverse(NewBins), lists:reverse(CDB)}; +compileEach(Config, [Source | Rest], Type, Env, {NewBins, CDB}) -> + Ext = filename:extension(Source), + Bin = replaceExtension(Source, Ext, ".o"), + Template = selectCompileTemplate(Type, compiler(Ext)), + Cmd = expandCommand(Template, Env, Source, Bin), + CDBEnt = cdbEntry(Source, Cmd, Rest), + NewCDB = [CDBEnt | CDB], + case needsCompile(Source, Bin) of + true -> + ShOpts = [{env, Env}, return_on_error, {use_stdout, false}], + execCompiler(Config, Source, Cmd, ShOpts), + compileEach(Config, Rest, Type, Env, {[Bin | NewBins], NewCDB}); + false -> + ?INFO("Skipping ~s\n", [Source]), + compileEach(Config, Rest, Type, Env, {NewBins, NewCDB}) + end. + +%% Generate a clang compilation db entry for Src and Cmd +cdbEntry(Src, Cmd, SrcRest) -> + %% Omit all variables from cmd, and use that as cmd in + %% CDB, because otherwise clang-* will complain about it. + CDBCmd = string:join( + lists:filter( + fun("$" ++ _) -> false; + (_) -> true + end, + string:tokens(Cmd, " ")), + " "), + + Cwd = rebarUtils:getCwd(), + %% If there are more source files, make sure we end the CDB entry + %% with a comma. + Sep = case SrcRest of + [] -> "~n"; + _ -> ",~n" + end, + %% CDB entry + ?FMT("{ \"file\" : ~p~n" + ", \"directory\" : ~p~n" + ", \"command\" : ~p~n" + "}~s", + [Src, Cwd, CDBCmd, Sep]). + +execCompiler(Config, Source, Cmd, ShOpts) -> + case rebarUtils:sh(Cmd, ShOpts) of + {error, {_RC, RawError}} -> + AbsSource = + case rebarUtils:processingBaseDir(Config) of + true -> + Source; + false -> + filename:absname(Source) + end, + ?CONSOLE("Compiling ~s\n", [AbsSource]), + Error = re:replace(RawError, Source, AbsSource, + [{return, list}, global]), + ?CONSOLE("~s", [Error]), + ?FAIL; + {ok, Output} -> + ?CONSOLE("Compiling ~s\n", [Source]), + ?CONSOLE("~s", [Output]) + end. + +needsCompile(Source, Bin) -> + needsLink(Bin, [Source | binDeps(Bin)]). + +%% NOTE: This relies on -MMD being passed to the compiler and returns an +%% empty list if the .d file is not available. This means header deps are +%% ignored on win32. +binDeps(Bin) -> + [DepFile] = portDeps([Bin]), + case file:read_file(DepFile) of + {ok, Deps} -> + Ds = parseBinDeps(list_to_binary(Bin), Deps), + Ds; + {error, _Err} -> + [] + end. + +parseBinDeps(Bin, Deps) -> + Sz = size(Bin), + <> = Deps, + Ds = re:split(X, "\\s*\\\\\\R\\s*|\\s+", [{return, binary}]), + [D || D <- Ds, D =/= <<>>]. + +needsLink(SoName, []) -> + filelib:last_modified(SoName) == 0; +needsLink(SoName, NewBins) -> + MaxLastMod = lists:max([filelib:last_modified(B) || B <- NewBins]), + case filelib:last_modified(SoName) of + 0 -> + true; + Other -> + MaxLastMod >= Other + end. + +%% +%% == port_specs == +%% + +getSpecs(Config, AppFile) -> + Specs = + case rebarConfig:getLocal(Config, port_specs, []) of + [] -> + %% No spec provided. Construct a spec + %% from old-school so_name and sources + [portSpecFromLegacy(Config, AppFile)]; + PortSpecs -> + Filtered = filterPortSpecs(PortSpecs), + OsType = os:type(), + [getPortSpec(Config, OsType, Spec) || Spec <- Filtered] + end, + [S || S <- Specs, S#spec.sources /= []]. + +portSpecFromLegacy(Config, AppFile) -> + %% Get the target from the so_name variable + Target = + case rebarConfig:get(Config, so_name, undefined) of + undefined -> + %% Generate a sensible default from app file + {_, AppName} = rebarUtils:appName(Config, AppFile), + filename:join("priv", lists:concat([AppName, "_drv.so"])); + AName -> + %% Old form is available -- use it + filename:join("priv", AName) + end, + %% Get the list of source files from port_sources + Sources = portSources(rebarConfig:getList(Config, port_sources, ["c_src/*.c"])), + #spec{ + type = targetType(Target), + link_lang = cc, + target = maybeSwitchExtension(os:type(), Target), + sources = Sources, + objects = portObjects(Sources) + }. + +filterPortSpecs(Specs) -> + [S || S <- Specs, filterPortSpec(S)]. + +filterPortSpec({ArchRegex, _, _, _}) -> + rebarUtils:isArch(ArchRegex); +filterPortSpec({ArchRegex, _, _}) -> + rebarUtils:isArch(ArchRegex); +filterPortSpec({_, _}) -> + true. + +getPortSpec(Config, OsType, {Target, Sources}) -> + getPortSpec(Config, OsType, {undefined, Target, Sources, []}); +getPortSpec(Config, OsType, {Arch, Target, Sources}) -> + getPortSpec(Config, OsType, {Arch, Target, Sources, []}); +getPortSpec(Config, OsType, {_Arch, Target, Sources, Opts}) -> + SourceFiles = portSources(Sources), + LinkLang = + case lists:any( + fun(Src) -> compiler(filename:extension(Src)) == "$CXX" end, + SourceFiles) + of + true -> cxx; + false -> cc + end, + ObjectFiles = portObjects(SourceFiles), + #spec{ + type = targetType(Target), + target = maybeSwitchExtension(OsType, Target), + link_lang = LinkLang, + sources = SourceFiles, + objects = ObjectFiles, + opts = portOpts(Config, Opts) + }. + +portSources(Sources) -> + lists:flatmap(fun filelib:wildcard/1, Sources). + +portObjects(SourceFiles) -> + [replaceExtension(O, ".o") || O <- SourceFiles]. + +portDeps(SourceFiles) -> + [replaceExtension(O, ".d") || O <- SourceFiles]. + +portOpts(Config, Opts) -> + [portOpt(Config, O) || O <- Opts]. + +portOpt(Config, {env, Env}) -> + {env, setupEnv(Config, Env)}; +portOpt(_Config, Opt) -> + Opt. + +maybeSwitchExtension({win32, nt}, Target) -> + switchToDllOrExe(Target); +maybeSwitchExtension(_OsType, Target) -> + Target. + +switchToDllOrExe(Target) -> + case filename:extension(Target) of + ".so" -> filename:rootname(Target, ".so") ++ ".dll"; + [] -> Target ++ ".exe"; + _Other -> Target + end. + + +%% == port_env == +%% Choose a compiler variable, based on a provided extension + +compiler(".cc") -> "$CXX"; +compiler(".cp") -> "$CXX"; +compiler(".cxx") -> "$CXX"; +compiler(".cpp") -> "$CXX"; +compiler(".CPP") -> "$CXX"; +compiler(".c++") -> "$CXX"; +compiler(".C") -> "$CXX"; +compiler(_) -> "$CC". + + +%% Given a list of {Key, Value} variables, and another list of default +%% {Key, Value} variables, return a merged list where the rule is if the +%% default is expandable expand it with the value of the variable list, +%% otherwise just return the value of the variable. + +applyDefaults(Vars, Defaults) -> + dict:to_list( + dict:merge( + fun(Key, VarValue, DefaultValue) -> + case isExpandable(DefaultValue) of + true -> + rebarUtils:expandEnvVariable(DefaultValue, + Key, + VarValue); + false -> VarValue + end + end, + dict:from_list(Vars), + dict:from_list(Defaults))). + +%% +%% Given a list of {Key, Value} environment variables, where Key may be defined +%% multiple times, walk the list and expand each self-reference so that we +%% end with a list of each variable singly-defined. +%% +mergeEachVar([], Vars) -> + Vars; +mergeEachVar([{Key, Value} | Rest], Vars) -> + Evalue = + case orddict:find(Key, Vars) of + error -> + %% Nothing yet defined for this key/value. + %% Expand any self-references as blank. + rebarUtils:expandEnvVariable(Value, Key, ""); + {ok, Value0} -> + %% Use previous definition in expansion + rebarUtils:expandEnvVariable(Value, Key, Value0) + end, + mergeEachVar(Rest, orddict:store(Key, Evalue, Vars)). + +%% +%% Give a unique list of {Key, Value} environment variables, expand each one +%% for every other key until no further expansions are possible. +%% +expandVarsLoop(Vars) -> + expandVarsLoop(Vars, [], dict:from_list(Vars), 10). + +expandVarsLoop(_Pending, _Recurse, _Vars, 0) -> + ?ABORT("Max. expansion reached for ENV vars!\n", []); +expandVarsLoop([], [], Vars, _Count) -> + lists:keysort(1, dict:to_list(Vars)); +expandVarsLoop([], Recurse, Vars, Count) -> + expandVarsLoop(Recurse, [], Vars, Count - 1); +expandVarsLoop([{K, V} | Rest], Recurse, Vars, Count) -> + %% Identify the variables that need expansion in this value + ReOpts = [global, {capture, all_but_first, list}, unicode], + case re:run(V, "\\\${?(\\w+)}?", ReOpts) of + {match, Matches} -> + %% Identify the unique variables that need to be expanded + UniqueMatches = lists:usort([M || [M] <- Matches]), + + %% For each variable, expand it and return the final + %% value. Note that if we have a bunch of unresolvable + %% variables, nothing happens and we don't bother + %% attempting further expansion + case expandKeysInValue(UniqueMatches, V, Vars) of + V -> + %% No change after expansion; move along + expandVarsLoop(Rest, Recurse, Vars, Count); + Expanded -> + %% Some expansion occurred; move to next k/v but + %% revisit this value in the next loop to check + %% for further expansion + NewVars = dict:store(K, Expanded, Vars), + expandVarsLoop(Rest, [{K, Expanded} | Recurse], + NewVars, Count) + end; + + nomatch -> + %% No values in this variable need expansion; move along + expandVarsLoop(Rest, Recurse, Vars, Count) + end. + +expandKeysInValue([], Value, _Vars) -> + Value; +expandKeysInValue([Key | Rest], Value, Vars) -> + NewValue = + case dict:find(Key, Vars) of + {ok, KValue} -> + rebarUtils:expandEnvVariable(Value, Key, KValue); + error -> + Value + end, + expandKeysInValue(Rest, NewValue, Vars). + +expandCommand(TmplName, Env, InFiles, OutFile) -> + Cmd0 = proplists:get_value(TmplName, Env), + Cmd1 = rebarUtils:expandEnvVariable(Cmd0, "PORT_IN_FILES", InFiles), + rebarUtils:expandEnvVariable(Cmd1, "PORT_OUT_FILE", OutFile). + +%% +%% Given a string, determine if it is expandable +%% +isExpandable(InStr) -> + case re:run(InStr, "\\\$", [{capture, none}]) of + match -> true; + nomatch -> false + end. + +%% +%% Filter a list of env vars such that only those which match the provided +%% architecture regex (or do not have a regex) are returned. +%% +filterEnv([], Acc) -> + lists:reverse(Acc); +filterEnv([{ArchRegex, Key, Value} | Rest], Acc) -> + case rebarUtils:isArch(ArchRegex) of + true -> + filterEnv(Rest, [{Key, Value} | Acc]); + false -> + filterEnv(Rest, Acc) + end; +filterEnv([{Key, Value} | Rest], Acc) -> + filterEnv(Rest, [{Key, Value} | Acc]). + +ertsDir() -> + lists:concat([code:root_dir(), "/erts-", erlang:system_info(version)]). + +osEnv() -> + ReOpts = [{return, list}, {parts, 2}, unicode], + Os = [list_to_tuple(re:split(S, "=", ReOpts)) || + S <- lists:filter(fun discardDepsVars/1, os:getenv())], + %% Drop variables without a name (win32) + [T1 || {K, _V} = T1 <- Os, K =/= []]. + +%% +%% To avoid having multiple repetitions of the same environment variables +%% (ERL_LIBS), avoid exporting any variables that may cause conflict with +%% those exported by the rebar_deps module (ERL_LIBS, REBAR_DEPS_DIR) +%% +discardDepsVars("ERL_LIBS=" ++ _Value) -> false; +discardDepsVars("REBAR_DEPS_DIR=" ++ _Value) -> false; +discardDepsVars(_Var) -> true. + +selectCompileTemplate(drv, Compiler) -> + selectCompileDrvTemplate(Compiler); +selectCompileTemplate(exe, Compiler) -> + selectCompileExeTemplate(Compiler). + +selectCompileDrvTemplate("$CC") -> "DRV_CC_TEMPLATE"; +selectCompileDrvTemplate("$CXX") -> "DRV_CXX_TEMPLATE". + +selectCompileExeTemplate("$CC") -> "EXE_CC_TEMPLATE"; +selectCompileExeTemplate("$CXX") -> "EXE_CXX_TEMPLATE". + +selectLinkTemplate(LinkLang, Target) -> + case {LinkLang, targetType(Target)} of + {cc, drv} -> "DRV_LINK_TEMPLATE"; + {cxx, drv} -> "DRV_LINK_CXX_TEMPLATE"; + {cc, exe} -> "EXE_LINK_TEMPLATE"; + {cxx, exe} -> "EXE_LINK_CXX_TEMPLATE" + end. + +targetType(Target) -> targetType_1(filename:extension(Target)). + +targetType_1(".so") -> drv; +targetType_1(".dll") -> drv; +targetType_1("") -> exe; +targetType_1(".exe") -> exe. + +erlInterfaceDir(Subdir) -> + case code:lib_dir(erl_interface, Subdir) of + {error, bad_name} -> + throw({error, {erl_interface, Subdir, "code:lib_dir(erl_interface)" + "is unable to find the erl_interface library."}}); + Dir -> Dir + end. + +defaultEnv() -> + Arch = os:getenv("REBAR_TARGET_ARCH"), + Vsn = os:getenv("REBAR_TARGET_ARCH_VSN"), + [ + {"CC", getTool(Arch, Vsn, "gcc", "cc")}, + {"CXX", getTool(Arch, Vsn, "g++", "c++")}, + {"AR", getTool(Arch, "ar", "ar")}, + {"AS", getTool(Arch, "as", "as")}, + {"CPP", getTool(Arch, Vsn, "cpp", "cpp")}, + {"LD", getTool(Arch, "ld", "ld")}, + {"RANLIB", getTool(Arch, Vsn, "ranlib", "ranlib")}, + {"STRIP", getTool(Arch, "strip", "strip")}, + {"NM", getTool(Arch, "nm", "nm")}, + {"OBJCOPY", getTool(Arch, "objcopy", "objcopy")}, + {"OBJDUMP", getTool(Arch, "objdump", "objdump")}, + + {"DRV_CXX_TEMPLATE", + "$CXX -c $CXXFLAGS $DRV_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, + {"DRV_CC_TEMPLATE", + "$CC -c $CFLAGS $DRV_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, + {"DRV_LINK_TEMPLATE", + "$CC $PORT_IN_FILES $LDFLAGS $DRV_LDFLAGS -o $PORT_OUT_FILE"}, + {"DRV_LINK_CXX_TEMPLATE", + "$CXX $PORT_IN_FILES $LDFLAGS $DRV_LDFLAGS -o $PORT_OUT_FILE"}, + {"EXE_CXX_TEMPLATE", + "$CXX -c $CXXFLAGS $EXE_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, + {"EXE_CC_TEMPLATE", + "$CC -c $CFLAGS $EXE_CFLAGS $PORT_IN_FILES -o $PORT_OUT_FILE"}, + {"EXE_LINK_TEMPLATE", + "$CC $PORT_IN_FILES $LDFLAGS $EXE_LDFLAGS -o $PORT_OUT_FILE"}, + {"EXE_LINK_CXX_TEMPLATE", + "$CXX $PORT_IN_FILES $LDFLAGS $EXE_LDFLAGS -o $PORT_OUT_FILE"}, + {"DRV_CFLAGS", "-g -Wall -fPIC -MMD $ERL_CFLAGS"}, + {"DRV_LDFLAGS", "-shared $ERL_LDFLAGS"}, + {"EXE_CFLAGS", "-g -Wall -fPIC -MMD $ERL_CFLAGS"}, + {"EXE_LDFLAGS", "$ERL_LDFLAGS"}, + + {"ERL_CFLAGS", lists:concat( + [ + " -I\"", erlInterfaceDir(include), + "\" -I\"", filename:join(ertsDir(), "include"), + "\" " + ])}, + {"ERL_EI_LIBDIR", lists:concat(["\"", erlInterfaceDir(lib), "\""])}, + {"ERL_LDFLAGS", " -L$ERL_EI_LIBDIR -lerl_interface -lei"}, + {"ERLANG_ARCH", rebarUtils:wordsize()}, + {"ERLANG_TARGET", rebarUtils:getArch()}, + + {"darwin", "DRV_LDFLAGS", + "-bundle -flat_namespace -undefined suppress $ERL_LDFLAGS"}, + + %% Solaris specific flags + {"solaris.*-64$", "CFLAGS", "-D_REENTRANT -m64 $CFLAGS"}, + {"solaris.*-64$", "CXXFLAGS", "-D_REENTRANT -m64 $CXXFLAGS"}, + {"solaris.*-64$", "LDFLAGS", "-m64 $LDFLAGS"}, + + %% OS X Leopard flags for 64-bit + {"darwin9.*-64$", "CFLAGS", "-m64 $CFLAGS"}, + {"darwin9.*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, + {"darwin9.*-64$", "LDFLAGS", "-arch x86_64 -flat_namespace -undefined suppress $LDFLAGS"}, + + %% OS X Lion onwards flags for 64-bit + {"darwin1[0-4].*-64$", "CFLAGS", "-m64 $CFLAGS"}, + {"darwin1[0-4].*-64$", "CXXFLAGS", "-m64 $CXXFLAGS"}, + {"darwin1[0-4].*-64$", "LDFLAGS", "-arch x86_64 -flat_namespace -undefined suppress $LDFLAGS"}, + + %% OS X Snow Leopard, Lion, and Mountain Lion flags for 32-bit + {"darwin1[0-2].*-32", "CFLAGS", "-m32 $CFLAGS"}, + {"darwin1[0-2].*-32", "CXXFLAGS", "-m32 $CXXFLAGS"}, + {"darwin1[0-2].*-32", "LDFLAGS", "-arch i386 -flat_namespace -undefined suppress $LDFLAGS"}, + + %% Windows specific flags + %% add MS Visual C++ support to rebar on Windows + {"win32", "CC", "cl.exe"}, + {"win32", "CXX", "cl.exe"}, + {"win32", "LINKER", "link.exe"}, + {"win32", "DRV_CXX_TEMPLATE", + %% DRV_* and EXE_* Templates are identical + "$CXX /c $CXXFLAGS $DRV_CFLAGS $PORT_IN_FILES /Fo$PORT_OUT_FILE"}, + {"win32", "DRV_CC_TEMPLATE", + "$CC /c $CFLAGS $DRV_CFLAGS $PORT_IN_FILES /Fo$PORT_OUT_FILE"}, + {"win32", "DRV_LINK_TEMPLATE", + "$LINKER $PORT_IN_FILES $LDFLAGS $DRV_LDFLAGS /OUT:$PORT_OUT_FILE"}, + {"win32", "DRV_LINK_CXX_TEMPLATE", + "$LINKER $PORT_IN_FILES $LDFLAGS $DRV_LDFLAGS /OUT:$PORT_OUT_FILE"}, + %% DRV_* and EXE_* Templates are identical + {"win32", "EXE_CXX_TEMPLATE", + "$CXX /c $CXXFLAGS $EXE_CFLAGS $PORT_IN_FILES /Fo$PORT_OUT_FILE"}, + {"win32", "EXE_CC_TEMPLATE", + "$CC /c $CFLAGS $EXE_CFLAGS $PORT_IN_FILES /Fo$PORT_OUT_FILE"}, + {"win32", "EXE_LINK_TEMPLATE", + "$LINKER $PORT_IN_FILES $LDFLAGS $EXE_LDFLAGS /OUT:$PORT_OUT_FILE"}, + {"win32", "EXE_LINK_CXX_TEMPLATE", + "$LINKER $PORT_IN_FILES $LDFLAGS $EXE_LDFLAGS /OUT:$PORT_OUT_FILE"}, + %% ERL_CFLAGS are ok as -I even though strictly it should be /I + {"win32", "ERL_LDFLAGS", + " /LIBPATH:$ERL_EI_LIBDIR erl_interface.lib ei.lib"}, + {"win32", "DRV_CFLAGS", "/Zi /Wall $ERL_CFLAGS"}, + {"win32", "DRV_LDFLAGS", "/DLL $ERL_LDFLAGS"}, + %% Provide some default Windows defines for convenience + {"win32", "CFLAGS", "/Wall /DWIN32 /D_WINDOWS /D_WIN32 /DWINDOWS /Ic_src $CFLAGS"}, + {"win32", "CXXFLAGS", "/Wall /DWIN32 /D_WINDOWS /D_WIN32 /DWINDOWS /Ic_src $CXXFLAGS"} + ]. + +getTool(Arch, Tool, Default) -> + getTool(Arch, false, Tool, Default). + +getTool(false, _, _, Default) -> Default; +getTool("", _, _, Default) -> Default; +getTool(Arch, false, Tool, _Default) -> Arch ++ "-" ++ Tool; +getTool(Arch, "", Tool, _Default) -> Arch ++ "-" ++ Tool; +getTool(Arch, Vsn, Tool, _Default) -> Arch ++ "-" ++ Tool ++ "-" ++ Vsn. diff --git a/src/rebarUtils.erl b/src/rebarUtils.erl new file mode 100644 index 0000000..72fedd0 --- /dev/null +++ b/src/rebarUtils.erl @@ -0,0 +1,520 @@ +-module(rebarUtils). + +-include("rebar.hrl"). + +-export([ + getCwd/0, + isArch/1, + getArch/0, + isAppDir/0, + appName/2, + wordsize/0, + sh/2, + abort/0, + abort/2, + expandEnvVariable/3, + delayedHalt/1, + baseDir/1, + processingBaseDir/1, + initVsnCache/1, + deleteEach/1 +]). + +getCwd() -> + {ok, Dir} = file:get_cwd(), + filename:join([Dir]). + +isArch(ArchRegex) -> + case re:run(getArch(), ArchRegex, [{capture, none}]) of + match -> + true; + nomatch -> + false + end. + + +%% REBAR_TARGET_ARCH, if used, should be set to the "standard" +%% target string. That is a prefix for binutils tools. +%% "x86_64-linux-gnu" or "arm-linux-gnueabi" are good candidates +%% ${REBAR_TARGET_ARCH}-gcc, ${REBAR_TARGET_ARCH}-ld ... + +getArch() -> + Arch = os:getenv("REBAR_TARGET_ARCH"), + Words = wordSize(Arch), + otpRelease() ++ "-" ++ get_system_arch(Arch) ++ "-" ++ Words. + +get_system_arch(Arch) when Arch =:= false; Arch =:= "" -> + erlang:system_info(system_architecture); +get_system_arch(Arch) -> + Arch. + +isAppDir() -> + isAppDir(rebarUtils:getCwd()). + +isAppDir(Dir) -> + SrcDir = filename:join([Dir, "src"]), + AppSrcScript = filename:join([SrcDir, "*.app.src.script"]), + AppSrc = filename:join([SrcDir, "*.app.src"]), + case {filelib:wildcard(AppSrcScript), filelib:wildcard(AppSrc)} of + {[AppSrcScriptFile], _} -> + {true, AppSrcScriptFile}; + {[], [AppSrcFile]} -> + {true, AppSrcFile}; + {[], []} -> + EbinDir = filename:join([Dir, "ebin"]), + App = filename:join([EbinDir, "*.app"]), + case filelib:wildcard(App) of + [AppFile] -> + {true, AppFile}; + [] -> + + false; + _ -> + ?ERROR("More than one .app file in ~s~n", [EbinDir]), + false + end; + {_, _} -> + ?ERROR("More than one .app.src file in ~s~n", [SrcDir]), + false + end. + + +appName(Config, AppFile) -> + case loadAppFile(Config, AppFile) of + {ok, NewConfig, AppName, _} -> + {NewConfig, AppName}; + {error, Reason} -> + ?ABORT("Failed to extract name from ~s: ~p\n", + [AppFile, Reason]) + end. + + +loadAppFile(Config, Filename) -> + AppFile = {app_file, Filename}, + case rebarConfig:getXconf(Config, {appfile, AppFile}, undefined) of + undefined -> + case consultAppFile(Filename) of + {ok, {application, AppName, AppData}} -> + Config1 = rebarConfig:setXconf(Config, + {appfile, AppFile}, + {AppName, AppData}), + {ok, Config1, AppName, AppData}; + {error, _} = Error -> + {error, {error, Error}}; + Other -> + {error, {unexpected_terms, Other}} + end; + {AppName, AppData} -> + {ok, Config, AppName, AppData} + end. + + +consultAppFile(Filename) -> + Result = case lists:suffix(".app", Filename) of + true -> + file:consult(Filename); + false -> + rebarConfig:consultFile(Filename) + end, + case Result of + {ok, [Term]} -> + {ok, Term}; + _ -> + Result + end. + +wordsize() -> + wordSize(os:getenv("REBAR_TARGET_ARCH")). + +%% +%% Options = [Option] -- defaults to [use_stdout, abort_on_error] +%% Option = ErrorOption | OutputOption | {cd, string()} | {env, Env} +%% ErrorOption = return_on_error | abort_on_error | {abort_on_error, string()} +%% OutputOption = use_stdout | {use_stdout, bool()} +%% Env = [{string(), Val}] +%% Val = string() | false +%% +sh(Command0, Options0) -> + ?INFO("sh info:\n\tcwd: ~p\n\tcmd: ~s\n", [getCwd(), Command0]), + + DefaultOptions = [use_stdout, abort_on_error], + Options = [expandShFlag(V) + || V <- proplists:compact(Options0 ++ DefaultOptions)], + + ErrorHandler = proplists:get_value(error_handler, Options), + OutputHandler = proplists:get_value(output_handler, Options), + + Command = patchOnWindows(Command0, proplists:get_value(env, Options, [])), + PortSettings = proplists:get_all_values(port_settings, Options) ++ + [exit_status, {line, 16384}, use_stdio, stderr_to_stdout, hide], + Port = open_port({spawn, Command}, PortSettings), + + case sh_loop(Port, OutputHandler, []) of + {ok, _Output} = Ok -> + Ok; + {error, {_Rc, _Output} = Err} -> + ErrorHandler(Command, Err) + end. + +-spec abort() -> no_return(). +abort() -> + throw(rebar_abort). + +-spec abort(string(), [term()]) -> no_return(). +abort(String, Args) -> + ?ERROR(String, Args), + abort(). + +%% +%% Given env. variable FOO we want to expand all references to +%% it in InStr. References can have two forms: $FOO and ${FOO} +%% The end of form $FOO is delimited with whitespace or eol +%% +expandEnvVariable(InStr, VarName, RawVarValue) -> + case string:chr(InStr, $$) of + 0 -> + %% No variables to expand + InStr; + _ -> + ReOpts = [global, unicode, {return, list}], + VarValue = re:replace(RawVarValue, "\\\\", "\\\\\\\\", ReOpts), + %% Use a regex to match/replace: + %% Given variable "FOO", match $FOO\W | $FOOeol | ${FOO}. + RegEx = io_lib:format("\\\$(~s(\\W|$)|{~s})", [VarName, VarName]), + re:replace(InStr, RegEx, [VarValue, "\\2"], ReOpts) + end. + +initVsnCache(Config) -> + initVsnCache(Config, os:getenv("REBAR_VSN_CACHE_FILE")). + +initVsnCache(Config, false) -> + rebarConfig:setXconf(Config, vsn_cache, dict:new()); +initVsnCache(Config, CacheFile) -> + {ok, CacheList} = file:consult(CacheFile), + CacheDict = dict:from_list(CacheList), + rebarConfig:setXconf(Config, vsn_cache, CacheDict). + +-spec delayedHalt(integer()) -> no_return(). +delayedHalt(Code) -> + %% Work around buffer flushing issue in erlang:halt if OTP older + %% than R15B01. + %% TODO: remove workaround once we require R15B01 or newer + %% R15B01 introduced erlang:halt/2 + case erlang:is_builtin(erlang, halt, 2) of + true -> + halt(Code); + false -> + case os:type() of + {win32, nt} -> + timer:sleep(100), + halt(Code); + _ -> + halt(Code), + %% workaround to delay exit until all output is written + receive after infinity -> ok end + end + end. + +deleteEach([]) -> + ok; +deleteEach([File | Rest]) -> + case file:delete(File) of + ok -> + deleteEach(Rest); + {error, enoent} -> + deleteEach(Rest); + {error, Reason} -> + ?ERROR("Failed to delete file ~s: ~p\n", [File, Reason]), + ?FAIL + end. + +baseDir(Config) -> + rebarConfig:getXconf(Config, base_dir). + +processingBaseDir(Config) -> + Cwd = rebarUtils:getCwd(), + processingBaseDir(Config, Cwd). + +processingBaseDir(Config, Dir) -> + AbsDir = filename:absname(Dir), + AbsDir =:= baseDir(Config). + +otpRelease() -> + case application:get_env(erlNpc, memoized_otp_release) of + {ok, Return} -> + Return; + undefined -> + Return = otpRelease_1(erlang:system_info(otp_release)), + application:set_env(erlNpc, memoized_otp_release, Return), + Return + end. + +%% If OTP <= R16, otp_release is already what we want. +otpRelease_1([$R, N | _] = Rel) when is_integer(N) -> + Rel; +%% If OTP >= 17.x, erlang:system_info(otp_release) returns just the +%% major version number, we have to read the full version from +%% a file. See http://www.erlang.org/doc/system_principles/versions.html +otpRelease_1(Rel) -> + Files = [ + filename:join([code:root_dir(), "releases", Rel, "OTP_VERSION"]), + filename:join([code:root_dir(), "OTP_VERSION"]) + ], + + %% It's possible that none of the above files exist on the filesystem, in + %% which case, we're just going to rely on the provided "Rel" (which should + %% just be the value of `erlang:system_info(otp_release)`). + case readOtpVersionFiles(Files) of + undefined -> + warnMissingOtpVersionFile(Rel), + Rel; + Vsn -> + Vsn + end. + +warnMissingOtpVersionFile(Rel) -> + ?WARN("No OTP_VERSION file found. Using version string ~p.~n", [Rel]). + +%% Try to open each file path provided, and if any of them exist on the +%% filesystem, read their contents and return the value of the first one found. +readOtpVersionFiles([]) -> + undefined; +readOtpVersionFiles([File | Rest]) -> + case file:read_file(File) of + {ok, Vsn} -> normalizeOtpVersion(Vsn); + {error, enoent} -> readOtpVersionFiles(Rest) + end. + +%% Takes the Version binary as read from the OTP_VERSION file and strips any +%% trailing "**" and trailing "\n", returning the string as a list. +normalizeOtpVersion(Vsn) -> + %% It's fine to rely on the binary module here because we can + %% be sure that it's available when the otp_release string does + %% not begin with $R. + Size = byte_size(Vsn), + %% The shortest vsn string consists of at least two digits + %% followed by "\n". Therefore, it's safe to assume Size >= 3. + case binary:part(Vsn, {Size, -3}) of + <<"**\n">> -> + %% The OTP documentation mentions that a system patched + %% using the otp_patch_apply tool available to licensed + %% customers will leave a '**' suffix in the version as a + %% flag saying the system consists of application versions + %% from multiple OTP versions. We ignore this flag and + %% drop the suffix, given for all intents and purposes, we + %% cannot obtain relevant information from it as far as + %% tooling is concerned. + binary:bin_to_list(Vsn, {0, Size - 3}); + _ -> + binary:bin_to_list(Vsn, {0, Size - 1}) + end. + +%% We do the shell variable substitution ourselves on Windows and hope that the +%% command doesn't use any other shell magic. +patchOnWindows(Cmd, Env) -> + case os:type() of + {win32, nt} -> + Cmd1 = "cmd /q /c " + ++ lists:foldl(fun({Key, Value}, Acc) -> + expandEnvVariable(Acc, Key, Value) + end, Cmd, Env), + %% Remove left-over vars + re:replace(Cmd1, "\\\$\\w+|\\\${\\w+}", "", + [global, {return, list}]); + _ -> + Cmd + end. + +expandShFlag(return_on_error) -> + {error_handler, + fun(_Command, Err) -> + {error, Err} + end}; +expandShFlag({abort_on_error, Message}) -> + {error_handler, + logMsgAndAbort(Message)}; +expandShFlag(abort_on_error) -> + {error_handler, + fun logAndAbort/2}; +expandShFlag(use_stdout) -> + {output_handler, + fun(Line, Acc) -> + ?CONSOLE("~s", [Line]), + [Line | Acc] + end}; +expandShFlag({use_stdout, false}) -> + {output_handler, + fun(Line, Acc) -> + [Line | Acc] + end}; +expandShFlag({cd, _CdArg} = Cd) -> + {port_settings, Cd}; +expandShFlag({env, _EnvArg} = Env) -> + {port_settings, Env}. + +-type err_handler() :: fun((string(), {integer(), string()}) -> no_return()). +-spec logMsgAndAbort(string()) -> err_handler(). +logMsgAndAbort(Message) -> + fun(_Command, {_Rc, _Output}) -> + ?ABORT(Message, []) + end. + +-spec logAndAbort(string(), {integer(), string()}) -> no_return(). +logAndAbort(Command, {Rc, Output}) -> + ?ABORT("sh(~s)~n" + "failed with return code ~w and the following output:~n" + "~s~n", [Command, Rc, Output]). + +sh_loop(Port, Fun, Acc) -> + receive + {Port, {data, {eol, Line}}} -> + sh_loop(Port, Fun, Fun(Line ++ "\n", Acc)); + {Port, {data, {noeol, Line}}} -> + sh_loop(Port, Fun, Fun(Line, Acc)); + {Port, {exit_status, 0}} -> + {ok, lists:flatten(lists:reverse(Acc))}; + {Port, {exit_status, Rc}} -> + {error, {Rc, lists:flatten(lists:reverse(Acc))}} + end. + +wordSize(Arch) when Arch =:= false; Arch =:= "" -> + nativeWordsize(); +wordSize(Arch) -> + AllArchs = [ + {"i686", "32"}, + {"i386", "32"}, + {"arm", "32"}, + {"aarch64", "64"}, + {"x86_64", "64"} + ], + case matchWordSize(Arch, AllArchs) of + false -> + case crossWordSize(Arch) of + "" -> + envWordSize(os:getenv("REBAR_TARGET_ARCH_WORDSIZE")); + WordSize -> + WordSize + end; + {_, Wordsize} -> + Wordsize + end. + +matchWordSize(Arch, [V = {Match, _Bits} | Vs]) -> + case re:run(Arch, Match, [{capture, none}]) of + match -> + V; + nomatch -> + matchWordSize(Arch, Vs) + end; +matchWordSize(_Arch, []) -> + false. + +envWordSize(Wordsize) when Wordsize =:= false; + Wordsize =:= "" -> + ?WARN("REBAR_TARGET_ARCH_WORDSIZE not set, assuming 32\n", []), + "32"; +envWordSize(Wordsize) -> + case Wordsize of + "16" -> Wordsize; + "32" -> Wordsize; + "64" -> Wordsize; + _ -> + ?WARN("REBAR_TARGET_ARCH_WORDSIZE bad value: ~p\n", [Wordsize]), + "32" + end. + +%% +%% Find out the word size of the target by using Arch-gcc +%% +crossWordSize(Arch) -> + crossSizeof(Arch, "void*"). + +%% +%% Find the size of target Type using a specially crafted C file +%% that will report an error on the line of the byte size of the type. +%% +crossSizeof(Arch, Type) -> + Compiler = if Arch =:= "" -> "cc"; + true -> Arch ++ "-gcc" + end, + TempFile = mkTempFile(".c"), + ok = file:write_file(TempFile, + <<"int t01 [1 - 2*(((long) (sizeof (TYPE))) == 1)];\n" + "int t02 [1 - 2*(((long) (sizeof (TYPE))) == 2)];\n" + "int t03 [1 - 2*(((long) (sizeof (TYPE))) == 3)];\n" + "int t04 [1 - 2*(((long) (sizeof (TYPE))) == 4)];\n" + "int t05 [1 - 2*(((long) (sizeof (TYPE))) == 5)];\n" + "int t06 [1 - 2*(((long) (sizeof (TYPE))) == 6)];\n" + "int t07 [1 - 2*(((long) (sizeof (TYPE))) == 7)];\n" + "int t08 [1 - 2*(((long) (sizeof (TYPE))) == 8)];\n" + "int t09 [1 - 2*(((long) (sizeof (TYPE))) == 9)];\n" + "int t10 [1 - 2*(((long) (sizeof (TYPE))) == 10)];\n" + "int t11 [1 - 2*(((long) (sizeof (TYPE))) == 11)];\n" + "int t12 [1 - 2*(((long) (sizeof (TYPE))) == 12)];\n" + "int t13 [1 - 2*(((long) (sizeof (TYPE))) == 13)];\n" + "int t14 [1 - 2*(((long) (sizeof (TYPE))) == 14)];\n" + "int t15 [1 - 2*(((long) (sizeof (TYPE))) == 15)];\n" + "int t16 [1 - 2*(((long) (sizeof (TYPE))) == 16)];\n" + >>), + Cmd = Compiler ++ " -DTYPE=\"" ++ Type ++ "\" " ++ TempFile, + ShOpts = [{use_stdout, false}, return_on_error], + {error, {_, Res}} = sh(Cmd, ShOpts), + ok = file:delete(TempFile), + case string:tokens(Res, ":") of + [_, Ln | _] -> + try list_to_integer(Ln) of + NumBytes -> integer_to_list(NumBytes * 8) + catch + error:_ -> + "" + end; + _ -> + "" + end. + +mkTempFile(Suffix) -> + {A, B, C} = rebarNow(), + Dir = tempDir(), + File = "rebar_" ++ os:getpid() ++ + integer_to_list(A) ++ "_" ++ + integer_to_list(B) ++ "_" ++ + integer_to_list(C) ++ Suffix, + filename:join(Dir, File). + +tempDir() -> + case os:type() of + {win32, _} -> windowsTempDir(); + _ -> "/tmp" + end. + +windowsTempDir() -> + case os:getenv("TEMP") of + false -> + case os:getenv("TMP") of + false -> "C:/WINDOWS/TEMP"; + TMP -> TMP + end; + TEMP -> TEMP + end. + +rebarNow() -> + case erlang:function_exported(erlang, timestamp, 0) of + true -> + apply(erlang, timestamp, []); + false -> + %% erlang:now/0 was deprecated in 18.0. One solution to avoid the + %% deprecation warning is to use + %% -compile({nowarn_deprecated_function, [{erlang, now, 0}]}), but + %% that would raise a warning in versions older than 18.0. Calling + %% erlang:now/0 via apply/3 avoids that. + apply(erlang, now, []) + end. + +nativeWordsize() -> + try erlang:system_info({wordsize, external}) of + Val -> + integer_to_list(8 * Val) + catch + error:badarg -> + integer_to_list(8 * erlang:system_info(wordsize)) + end.