From 9c15b57aa413a773192fdd0aa75f908a08a47b84 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 26 Sep 2020 22:47:03 -0400 Subject: [PATCH 1/2] Specify dependencies for MIB files This is required for compile order to be maintained in a post-3.14 world. --- src/rebar_compiler_mib.erl | 66 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/src/rebar_compiler_mib.erl b/src/rebar_compiler_mib.erl index 499976d3..e25cc809 100644 --- a/src/rebar_compiler_mib.erl +++ b/src/rebar_compiler_mib.erl @@ -61,8 +61,9 @@ atoms_in_mib_first_files_warning(Atoms) -> ?WARN(W, [Atoms]). -dependencies(_, _, _) -> - []. +dependencies(File, _Dir, SrcDirs) -> + SrcFiles = lists:append([src_files(SrcDir) || SrcDir <- SrcDirs]), + file_deps(File, SrcFiles). compile(Source, OutDirs, _, Opts) -> {_, BinOut} = lists:keyfind(".bin", 1, OutDirs), @@ -99,3 +100,64 @@ clean(MibFiles, AppInfo) -> rebar_file_utils:delete_each( [filename:join([AppDir, "include", MIB++".hrl"]) || MIB <- MIBs]), ok = rebar_file_utils:rm_rf(filename:join([AppDir, "priv/mibs/*.bin"])). + +src_files(Dir) -> + %% .mib extension is assumed to be valid here + case file:list_dir(Dir) of + {ok, Files} -> + [filename:join(Dir, File) + || File <- Files, + filename:extension(File) =:= ".mib"]; + _ -> + [] + end. + +file_deps(File, Files) -> + DepMods = imports_in_file(File), + lists:filter( + fun(F) -> + Mods = modules_in_file(F), + lists:any(fun(M) -> lists:member(M, Mods) end, DepMods) + end, + Files + ). + +modules_in_file(File) -> + {ok, Bin} = file:read_file(File), + Res = re:run( + Bin, + "^([a-zA-Z0-9_-]+)\\s+DEFINITIONS\\s+::=\\s+BEGIN", + [multiline, {capture, all_but_first, list}, global, unicode] + ), + case Res of + nomatch -> + []; + {match, List} -> + lists:usort(lists:append(List)) + end. + +imports_in_file(File) -> + {ok, Bin} = file:read_file(File), + ImportMatch = re:run( + Bin, + "IMPORTS\\s+(.*);", + [multiline, dotall, ungreedy, {capture, all_but_first, list}, global] + ), + case ImportMatch of + nomatch -> + []; + {match, ImportSections} -> + Modules = re:run( + ImportSections, + "FROM\\s+([a-zA-Z0-9_-]+)\\s+", + [multiline, {capture, all_but_first, list}, global] + ), + case Modules of + nomatch -> + []; + {match, List} -> + lists:usort(lists:append(List)) + end + end. + + From 8de300813d476131870fb78eae27f900a244ff1f Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 27 Sep 2020 09:13:46 -0400 Subject: [PATCH 2/2] add MIB deps handling test --- test/rebar_compile_SUITE.erl | 111 ++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/test/rebar_compile_SUITE.erl b/test/rebar_compile_SUITE.erl index d2717f54..6af1f09e 100644 --- a/test/rebar_compile_SUITE.erl +++ b/test/rebar_compile_SUITE.erl @@ -26,7 +26,8 @@ all() -> delete_beam_if_source_deleted, deps_in_path, checkout_priority, highest_version_of_pkg_dep, parse_transform_test, erl_first_files_test, mib_test, - umbrella_mib_first_test, only_default_transitive_deps, clean_all, + umbrella_mib_first_test, deps_mib_test, + only_default_transitive_deps, clean_all, clean_specific, profile_deps, deps_build_in_prod, only_deps, override_deps, git_subdir_deps, override_add_deps, override_del_deps, override_del_pkg_deps, override_opts, override_add_opts, override_del_opts, @@ -1465,6 +1466,114 @@ umbrella_mib_first_test(Config) -> %% check the mibs dir was linked into the _build dir true = filelib:is_dir(filename:join([AppsDir, "_build", "default", "lib", Name, "mibs"])). +deps_mib_test() -> + [{doc, "reproduces the dependency handling required for the issue " + "reported in https://github.com/erlang/rebar3/issues/2372"}]. +deps_mib_test(Config) -> + Priv = ?config(priv_dir, Config), + CliLvl2Mib = + "---\n" + "---\n" + "---\n" + "LVL2-MIB DEFINITIONS ::= BEGIN\n" + "IMPORTS\n" + " MODULE-IDENTITY, OBJECT-TYPE\n" + " FROM SNMPv2-SMI\n" + " lvlModules, lvlApplications\n" + " FROM LVL0-REG\n" + " LvlFoo\n" + " FROM LVL0-TC\n" + " ;\n" + "\n" + "lvl2Module MODULE-IDENTITY\n" + " LAST-UPDATED \"202009261630Z\"\n" + " ORGANIZATION \"'some org'\"\n" + " CONTACT-INFO \"'Contact: some contact'\"\n" + " DESCRIPTION\n" + " \" \"\n" + " ::= { lvlModules 3 }\n" + "\n\n\n" + "END", + CpiLvl0Mib1 = + " \n" + "---\n" + "---\n\n" + "LVL0-TC DEFINITIONS ::= BEGIN\n" + "IMPORTS\n" + " lvlModules\n" + " FROM LVL0-REG\n" + " MODULE-IDENTITY\n" + " FROM SNMPv2-SMI\n" + " TEXTUAL-CONVENTION\n" + " FROM SNMPv2-TC\n" + " ;\n\n" + "lvlTcModule MODULE-IDENTITY\n" + " LAST-UPDATED \"202009261630Z\"\n" + " ORGANIZATION \"'some org'\"\n" + " CONTACT-INFO \"'Contact: some contact'\"\n" + " DESCRIPTION\n" + " \" This MIB is part of the LVL MIB. It defines common\n" + " Textual Conventions used in other LVL mib modules.\"\n" + " ::= { lvlModules 2 }\n\n\n" + "LvlFoo ::= TEXTUAL-CONVENTION\n" + " DISPLAY-HINT \"512a\"\n" + " STATUS current\n" + " DESCRIPTION \"\"\n" + " SYNTAX OCTET STRING (SIZE (1..512))\n\n" + "LvlEnum ::= TEXTUAL-CONVENTION\n" + " STATUS current\n" + " DESCRIPTION \"\"\n" + " SYNTAX INTEGER { foo(1), bar(2), baz(3) }\n\n" + "END\n", + CpiLvl0Mib2 = + "---\n---\n---\n\n" + "LVL0-REG DEFINITIONS ::= BEGIN\n" + "\n" + "IMPORTS\n" + " MODULE-IDENTITY, enterprises\n" + " FROM SNMPv2-SMI\n" + " ;\n\n" + "lvlRegModule MODULE-IDENTITY\n" + " LAST-UPDATED \"202009261630Z\"\n" + " ORGANIZATION \"'some org'\"\n" + " CONTACT-INFO \"'Contact: some contact'\"\n" + " DESCRIPTION\n" + " \"The root MIB module for LVL\"\n" + " ::= { lvlModules 1 }\n\n" + "-- Example Enterprise Number for Documentation use\n" + "example OBJECT IDENTIFIER ::= { enterprises 32473 }\n" + "\n" + "lvl OBJECT IDENTIFIER ::= { example 1 }\n\n" + "-- sub-tree for registrations (Modules information)\n" + "lvlReg OBJECT IDENTIFIER ::= { lvl 1 }\n" + "lvlModules OBJECT IDENTIFIER ::= { lvlReg 1 }\n" + "\n" + "-- the application subtree\n" + "lvlApplications OBJECT IDENTIFIER ::= { lvl 2 }\n" + "END\n", + CliLvl2Path = filename:join([Priv, "deps_mib", "cli_lvl2", "mibs", "LVL2-MIB.mib"]), + CpiLvl0Path1 = filename:join([Priv, "deps_mib", "cpi_lvl0", "mibs", "LVL0-TC.mib"]), + CpiLvl0Path2 = filename:join([Priv, "deps_mib", "cpi_lvl0", "mibs", "LVL0-REG.mib"]), + NprLvl1Path = filename:join([Priv, "deps_mib", "npr_lvl1", "mibs"]), % no mibs dir + FakeLvl1Path = filename:join([Priv, "deps_mib", "fake_lvl1", "mibs", "no_file"]), % no mibs file + ok = filelib:ensure_dir(CliLvl2Path), + ok = filelib:ensure_dir(CpiLvl0Path1), % CpiLvl0Path2 is the same dir + ok = filelib:ensure_dir(NprLvl1Path), + ok = filelib:ensure_dir(FakeLvl1Path), + ok = file:write_file(CliLvl2Path, CliLvl2Mib), + ok = file:write_file(CpiLvl0Path1, CpiLvl0Mib1), + ok = file:write_file(CpiLvl0Path2, CpiLvl0Mib2), + Res = rebar_compiler_mib:dependencies( + CliLvl2Path, + filename:dirname(CliLvl2Path), + [filename:dirname(CliLvl2Path), + filename:dirname(CpiLvl0Path1), + NprLvl1Path, + filename:dirname(FakeLvl1Path)] + ), + ?assertEqual(lists:sort([CpiLvl0Path1, CpiLvl0Path2]), + lists:sort(Res)), + ok. only_default_transitive_deps(Config) -> AppDir = ?config(apps, Config),