%%%-------------------------------------------------------------------
%%% @author liushl
%%% @doc
%%%     从TXT文本读入敏感词库，输出生成data_filter.erl文件
%%%     TXT文本格式：一个词组一行
%%% @end
%%%-------------------------------------------------------------------
-module(word_filter).
-author("liushl").

-define(TXT_DIR,  "../src/tool/").                          %% 敏感词库txt文件目录
-define(TXT_FILE(FName), lists:concat([?TXT_DIR, FName])).  %% 敏感词库txt文件
-define(ERL_FILE, "../src/data/data_filter.erl").           %% 输出文件
%% 额外添加敏感词
-define(EXTRA_FILTER, []).
%% API
-export([start/0, start/1]).
-compile(export_all).
%% 支持版本及对应的文件
-define(TXT_FILES, [
    {"cn", "words.txt"}
    ,{"kr", "words_kr.txt"}
]).

start() ->
    usage(),
    case io:fread("\nInput version > ", "~ts") of
        {ok, [Input | _]} ->
            Ver =
                case Input of
                    "" -> "cn";
                    _  -> Input
                end,
            start(Ver);
        _Err ->
            io:format("Input error : ~p~n", [_Err])
    end.

start(Ver) ->
    case lists:keyfind(Ver, 1, ?TXT_FILES) of
        {_, FName} ->
            InFile = ?TXT_FILE(FName),
            conv(InFile);
        false ->
            io:format("No version file [~ts]\n", [util:make_sure_list(Ver)]),
            usage()
    end.

usage() ->
    Keys = [Key || {Key, _} <- ?TXT_FILES],
    KeysStr = string:join(Keys, " | "),
    io:format("Usage:\n"
    " ~w:start() | ~w:start(Version)\n"
    "  optional version: ~ts\n",
        [?MODULE, ?MODULE, KeysStr]
    ).

conv(InFile) ->
    {ok, R} = file:open(InFile, [read, {encoding, utf8}]),
    AllWords = read_file(R, ?EXTRA_FILTER),
    %% 将词按照字符长度排序
    SortWords = lists:sort(fun sort_words/2, AllWords),
    AfterSort = [One || {One, _} <-  SortWords],
    file:close(R),
    io:format("AllWords:~w~n", [length(AllWords)]),
    {ok, W} = file:open(?ERL_FILE, [write, {encoding, utf8}]),

    io:format(W, "~s~n", ["-module(data_filter)."]),
    io:format(W, "~s~n", ["-compile(export_all)."]),
    io:format(W, "~n~n", []),
    io:format(W, "~s~n", ["talk() -> "]),
    io:format(W, "~s~n", ["["]),
    AfterFilter = filter(W, AfterSort),
    io:format("AfterFilter:~w~n", [length(AfterFilter)]),
    io:format(W, "~s~n", ["]."]),
    file:close(W),
    io:format("Done, Success!~n").

%% 按长度排序，以便于使剩余结果最少而达到过滤效果
%% 再按字符顺序排序，以便于得到稳定的排序结果，
%% 避免增加一些词汇时，生成结果可能差异非常大，难以对比检查
sort_words({WordA, LenA}, {WordB, LenB}) ->
    if
        LenA < LenB -> true;
        LenA > LenB -> false;
        true ->
            WordA =< WordB
    end.

%% 读取屏蔽词
read_file(Io, Content) ->
    Extra = [{One, length(One)} || One <- Content],
    read_file_do(Io, Extra).
read_file_do(Io, Content) ->
    case io:get_line(Io, "") of
        eof -> Content;
        _Word ->
            %% 去掉换行符/去掉两边的空格/去掉空行/去掉标题
            case string:tokens(_Word, "\n\r") of
                [Word1|_] ->
                    Word2 = string:strip(Word1),
                    case check_not_title(Word2)
                        andalso check_not_empty(Word2)
                    of
                        true ->
                            OneS = unicode:characters_to_list(Word2),
                            read_file_do(Io, [{OneS, length(OneS)}|Content]);
                        false ->
                            read_file_do(Io, Content)
                    end;
                _ ->
                    read_file_do(Io, Content)
            end
    end.
filter(S, [H|T]) ->
    io:format(S, "~ts~n", ["\""++H++"\""]),
    filter(S, [H], T);
filter(_, _) -> [].

%% 如果已入词库中有词是待检测词的子串，则抛弃改待检测词
filter(_S, OkList, []) ->
    OkList;
filter(S, OkList, [ToCheck|Rest]) ->
    case check_no_sub_list(OkList, ToCheck) of
        true  ->
            io:format(S, "~ts~n", ["\," ++ "\""++ToCheck++"\""]),
            filter(S, [ToCheck|OkList], Rest);
        false ->
            filter(S, OkList, Rest)
    end.

%% 如果待检测词是已有词库的字串吗，则不要
check_no_sub_list([One|Rest], Two) ->
    case string:str(Two, One) of
        0 -> check_no_sub_list(Rest, Two);
        _ ->
            false
    end;
check_no_sub_list([], _) -> true.

%% 如果是#开头的，则认为是标题，不要
check_not_title(String) ->
    case String of
        [$#|_] ->
            false;
        _ ->
            true
    end.

%% 如果是空行，则不要
check_not_empty(String) ->
    case String of
        "" -> false;
        _  -> true
    end.