From a7c15eaccfb835c3072c528caf7173a0e3bd7ac7 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 17 Apr 2025 10:06:06 +0200 Subject: [PATCH 01/27] mod_antispam: initial import from ejabberd-contrib/mod_spam_filter --- src/mod_antispam.erl | 1072 +++++++++++++++++++++++++++++++++++++ src/mod_antispam_rtbl.erl | 132 +++++ 2 files changed, 1204 insertions(+) create mode 100644 src/mod_antispam.erl create mode 100644 src/mod_antispam_rtbl.erl diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl new file mode 100644 index 000000000..986f21066 --- /dev/null +++ b/src/mod_antispam.erl @@ -0,0 +1,1072 @@ +%%%---------------------------------------------------------------------- +%%% File : mod_antispam.erl +%%% Author : Holger Weiss +%%% Author : Stefan Strigler +%%% Purpose : Filter spam messages based on sender JID and content +%%% Created : 31 Mar 2019 by Holger Weiss +%%% +%%% +%%% ejabberd, Copyright (C) 2019-2025 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%---------------------------------------------------------------------- + +-module(mod_antispam). +-author('holger@zedat.fu-berlin.de'). +-author('stefan@strigler.de'). + +-behaviour(gen_server). +-behaviour(gen_mod). + +%% gen_mod callbacks. +-export([start/2, + stop/1, + reload/3, + depends/2, + mod_doc/0, + mod_opt_type/1, + mod_options/1]). + +%% gen_server callbacks. +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +%% ejabberd_hooks callbacks. +-export([s2s_in_handle_info/2, + s2s_receive_packet/1, + sm_receive_packet/1, + reopen_log/0]). + +%% ejabberd_commands callbacks. +-export([add_blocked_domain/2, + add_to_spam_filter_cache/2, + drop_from_spam_filter_cache/2, + expire_spam_filter_cache/2, + get_blocked_domains/1, + get_commands_spec/0, + get_spam_filter_cache/1, + reload_spam_filter_files/1, + remove_blocked_domain/2]). + +-include("ejabberd_commands.hrl"). +-include("logger.hrl"). + +-include_lib("xmpp/include/xmpp.hrl"). + +-type url() :: binary(). +-type filename() :: binary() | none. +-type jid_set() :: sets:set(ljid()). +-type url_set() :: sets:set(url()). +-type s2s_in_state() :: ejabberd_s2s_in:state(). + +-record(state, + {host = <<>> :: binary(), + dump_fd = undefined :: file:io_device() | undefined, + url_set = sets:new() :: url_set(), + jid_set = sets:new() :: jid_set(), + jid_cache = #{} :: map(), + max_cache_size = 0 :: non_neg_integer() | unlimited, + rtbl_host = none :: binary() | none, + rtbl_subscribed = false :: boolean(), + rtbl_retry_timer = undefined :: reference() | undefined, + rtbl_domains_node :: binary(), + blocked_domains = #{} :: #{binary() => any()}, + whitelist_domains = #{} :: #{binary() => false} + }). + +-type state() :: #state{}. + +-define(COMMAND_TIMEOUT, timer:seconds(30)). +-define(HTTPC_TIMEOUT, timer:seconds(3)). +-define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). +-define(DEFAULT_CACHE_SIZE, 10000). + +%%-------------------------------------------------------------------- +%% gen_mod callbacks. +%%-------------------------------------------------------------------- +-spec start(binary(), gen_mod:opts()) -> ok | {error, any()}. +start(Host, Opts) -> + case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of + false -> + ejabberd_commands:register_commands(?MODULE, get_commands_spec()); + true -> + ok + end, + gen_mod:start_child(?MODULE, Host, Opts). + +-spec stop(binary()) -> ok | {error, any()}. +stop(Host) -> + case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of + false -> + ejabberd_commands:unregister_commands(get_commands_spec()); + true -> + ok + end, + gen_mod:stop_child(?MODULE, Host). + +-spec reload(binary(), gen_mod:opts(), gen_mod:opts()) -> ok. +reload(Host, NewOpts, OldOpts) -> + ?DEBUG("reloading", []), + Proc = get_proc_name(Host), + gen_server:cast(Proc, {reload, NewOpts, OldOpts}). + +-spec depends(binary(), gen_mod:opts()) -> [{module(), hard | soft}]. +depends(_Host, _Opts) -> + [{mod_pubsub, soft}]. + +-spec mod_opt_type(atom()) -> econf:validator(). +mod_opt_type(spam_domains_file) -> + econf:either( + econf:enum([none]), + econf:file()); +mod_opt_type(whitelist_domains_file) -> + econf:either( + none, + econf:binary()); +mod_opt_type(spam_dump_file) -> + econf:either( + econf:enum([none]), + econf:binary()); +mod_opt_type(spam_jids_file) -> + econf:either( + econf:enum([none]), + econf:file()); +mod_opt_type(spam_urls_file) -> + econf:either( + econf:enum([none]), + econf:file()); +mod_opt_type(access_spam) -> + econf:acl(); +mod_opt_type(cache_size) -> + econf:pos_int(unlimited); +mod_opt_type(rtbl_host) -> + econf:either( + econf:enum([none]), + econf:host()); +mod_opt_type(rtbl_domains_node) -> + econf:non_empty(econf:binary()). + +-spec mod_options(binary()) -> [{atom(), any()}]. +mod_options(_Host) -> + [{spam_domains_file, none}, + {spam_dump_file, none}, + {spam_jids_file, none}, + {spam_urls_file, none}, + {whitelist_domains_file, none}, + {access_spam, none}, + {cache_size, ?DEFAULT_CACHE_SIZE}, + {rtbl_host, none}, + {rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}]. + +mod_doc() -> #{}. + +%%-------------------------------------------------------------------- +%% gen_server callbacks. +%%-------------------------------------------------------------------- +-spec init(list()) -> {ok, state()} | {stop, term()}. +init([Host, Opts]) -> + process_flag(trap_exit, true), + DumpFile = expand_host(gen_mod:get_opt(spam_dump_file, Opts), Host), + Files = #{domains => gen_mod:get_opt(spam_domains_file, Opts), + jid => gen_mod:get_opt(spam_jids_file, Opts), + url => gen_mod:get_opt(spam_urls_file, Opts), + whitelist_domains => gen_mod:get_opt(whitelist_domains_file, Opts)}, + try read_files(Files) of + #{jid := JIDsSet, url := URLsSet, domains := SpamDomainsSet, whitelist_domains := WhitelistDomains} -> + ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, + s2s_in_handle_info, 90), + ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, + s2s_receive_packet, 50), + ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, + sm_receive_packet, 50), + ejabberd_hooks:add(reopen_log_hook, ?MODULE, + reopen_log, 50), + ejabberd_hooks:add(local_send_to_resource_hook, Host, + mod_antispam_rtbl, pubsub_event_handler, 50), + RTBLHost = gen_mod:get_opt(rtbl_host, Opts), + RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), + InitState0 = #state{host = Host, + jid_set = JIDsSet, + url_set = URLsSet, + max_cache_size = gen_mod:get_opt(cache_size, Opts), + blocked_domains = set_to_map(SpamDomainsSet), + whitelist_domains = set_to_map(WhitelistDomains, false), + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode}, + mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), + InitState = init_open_dump_file(DumpFile, InitState0), + {ok, InitState} + catch {Op, File, Reason} when Op == open; + Op == read -> + ?CRITICAL_MSG("Cannot ~s ~s: ~s", [Op, File, format_error(Reason)]), + {stop, config_error} + end. + +init_open_dump_file(none, State) -> + State; +init_open_dump_file(DumpFile, State) -> + case filelib:ensure_dir(DumpFile) of + ok -> + ok; + {error, Reason} -> + Dirname = filename:dirname(DumpFile), + throw({open, Dirname, Reason}) + end, + open_dump_file(DumpFile, State). + +-spec handle_call(term(), {pid(), term()}, state()) + -> {reply, {spam_filter, term()}, state()} | {noreply, state()}. +handle_call({check_jid, From}, _From, #state{jid_set = JIDsSet} = State) -> + {Result, State1} = filter_jid(From, JIDsSet, State), + {reply, {spam_filter, Result}, State1}; +handle_call({check_body, URLs, JIDs, From}, _From, + #state{url_set = URLsSet, jid_set = JIDsSet} = State) -> + {Result1, State1} = filter_body(URLs, URLsSet, From, State), + {Result2, State2} = filter_body(JIDs, JIDsSet, From, State1), + Result = if Result1 == spam -> + Result1; + true -> + Result2 + end, + {reply, {spam_filter, Result}, State2}; +handle_call({resolve_redirects, URLs}, _From, State) -> + ResolvedURLs = do_resolve_redirects(URLs, []), + {reply, {spam_filter, ResolvedURLs}, State}; +handle_call({reload_files, Files}, _From, State) -> + {Result, State1} = reload_files(Files, State), + {reply, {spam_filter, Result}, State1}; +handle_call({expire_cache, Age}, _From, State) -> + {Result, State1} = expire_cache(Age, State), + {reply, {spam_filter, Result}, State1}; +handle_call({add_to_cache, JID}, _From, State) -> + {Result, State1} = add_to_cache(JID, State), + {reply, {spam_filter, Result}, State1}; +handle_call({drop_from_cache, JID}, _From, State) -> + {Result, State1} = drop_from_cache(JID, State), + {reply, {spam_filter, Result}, State1}; +handle_call(get_cache, _From, #state{jid_cache = Cache} = State) -> + {reply, {spam_filter, maps:to_list(Cache)}, State}; +handle_call({add_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains} = State) -> + BlockedDomains1 = maps:merge(BlockedDomains, #{Domain => true}), + Txt = format("~s added to blocked domains", [Domain]), + {reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}}; +handle_call({remove_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains} = State) -> + BlockedDomains1 = maps:remove(Domain, BlockedDomains), + Txt = format("~s removed from blocked domains", [Domain]), + {reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}}; +handle_call(get_blocked_domains, _From, #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = State) -> + {reply, {blocked_domains, maps:merge(BlockedDomains, WhitelistDomains)}, State}; +handle_call({is_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = State) -> + {reply, maps:get(Domain, maps:merge(BlockedDomains, WhitelistDomains), false) =/= false, State}; +handle_call(Request, From, State) -> + ?ERROR_MSG("Got unexpected request from ~p: ~p", [From, Request]), + {noreply, State}. + +-spec handle_cast(term(), state()) -> {noreply, state()}. +handle_cast({dump, _XML}, #state{dump_fd = undefined} = State) -> + {noreply, State}; +handle_cast({dump, XML}, #state{dump_fd = Fd} = State) -> + case file:write(Fd, [XML, <<$\n>>]) of + ok -> + ok; + {error, Reason} -> + ?ERROR_MSG("Cannot write spam to dump file: ~s", + [file:format_error(Reason)]) + end, + {noreply, State}; +handle_cast({reload, NewOpts, OldOpts}, + #state{host = Host, + rtbl_host = OldRTBLHost, + rtbl_domains_node = OldRTBLDomainsNode, + rtbl_retry_timer = RTBLRetryTimer} = State) -> + misc:cancel_timer(RTBLRetryTimer), + State1 = case {gen_mod:get_opt(spam_dump_file, OldOpts), + gen_mod:get_opt(spam_dump_file, NewOpts)} of + {OldDumpFile, NewDumpFile} when NewDumpFile /= OldDumpFile -> + close_dump_file(expand_host(OldDumpFile, Host), State), + open_dump_file(expand_host(NewDumpFile, Host), State); + {_OldDumpFile, _NewDumpFile} -> + State + end, + State2 = case {gen_mod:get_opt(cache_size, OldOpts), + gen_mod:get_opt(cache_size, NewOpts)} of + {OldMax, NewMax} when NewMax < OldMax -> + shrink_cache(State1#state{max_cache_size = NewMax}); + {OldMax, NewMax} when NewMax > OldMax -> + State1#state{max_cache_size = NewMax}; + {_OldMax, _NewMax} -> + State1 + end, + ok = mod_antispam_rtbl:unsubscribe(OldRTBLHost, OldRTBLDomainsNode, Host), + Files = #{domains => gen_mod:get_opt(spam_domains_file, NewOpts), + jid => gen_mod:get_opt(spam_jids_file, NewOpts), + url => gen_mod:get_opt(spam_urls_file, NewOpts), + whitelist_domains => gen_mod:get_opt(whitelist_domains_file, NewOpts)}, + {_Result, State3} = reload_files(Files, State2#state{blocked_domains = #{}}), + RTBLHost = gen_mod:get_opt(rtbl_host, NewOpts), + RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, NewOpts), + ok = mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), + {noreply, State3#state{rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode}}; +handle_cast(reopen_log, State) -> + {noreply, reopen_dump_file(State)}; +handle_cast({update_blocked_domains, NewItems}, #state{blocked_domains = BlockedDomains} = State) -> + {noreply, State#state{blocked_domains = maps:merge(BlockedDomains, NewItems)}}; +handle_cast(Request, State) -> + ?ERROR_MSG("Got unexpected request from: ~p", [Request]), + {noreply, State}. + +-spec handle_info(term(), state()) -> {noreply, state()}. +handle_info({iq_reply, timeout, blocked_domains}, State) -> + ?WARNING_MSG("Fetching blocked domains failed: fetch timeout. Retrying in 60 seconds", []), + {noreply, State#state{rtbl_retry_timer = erlang:send_after(60000, self(), request_blocked_domains)}}; +handle_info({iq_reply, #iq{type = error} = IQ, blocked_domains}, State) -> + ?WARNING_MSG("Fetching blocked domains failed: ~p. Retrying in 60 seconds", + [xmpp:format_stanza_error(xmpp:get_error(IQ))]), + {noreply, State#state{rtbl_retry_timer = erlang:send_after(60000, self(), request_blocked_domains)}}; +handle_info({iq_reply, IQReply, blocked_domains}, + #state{blocked_domains = OldBlockedDomains, + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode, + host = Host} = State) -> + case mod_antispam_rtbl:parse_blocked_domains(IQReply) of + undefined -> + ?WARNING_MSG("Fetching initial list failed: invalid result payload", []), + {noreply, State#state{rtbl_retry_timer = undefined}}; + NewBlockedDomains -> + ok = mod_antispam_rtbl:subscribe(RTBLHost, RTBLDomainsNode, Host), + {noreply, State#state{rtbl_retry_timer = undefined, + rtbl_subscribed = true, + blocked_domains = maps:merge(OldBlockedDomains, NewBlockedDomains)}} + end; +handle_info({iq_reply, timeout, subscribe_result}, State) -> + ?WARNING_MSG("Subscription error: request timeout", []), + {noreply, State#state{rtbl_subscribed = false}}; +handle_info({iq_reply, #iq{type = error} = IQ, subscribe_result}, State) -> + ?WARNING_MSG("Subscription error: ~p", [xmpp:format_stanza_error(xmpp:get_error(IQ))]), + {noreply, State#state{rtbl_subscribed = false}}; +handle_info({iq_reply, IQReply, subscribe_result}, State) -> + ?DEBUG("Got subscribe result: ~p", [IQReply]), + {noreply, State#state{rtbl_subscribed = true}}; +handle_info({iq_reply, _IQReply, unsubscribe_result}, State) -> + %% FIXME: we should check it's true (of type `result`, not `error`), but at that point, what + %% would we do? + {noreply, State#state{rtbl_subscribed = false}}; +handle_info(request_blocked_domains, #state{host = Host, rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode} = State) -> + mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), + {noreply, State}; +handle_info(Info, State) -> + ?ERROR_MSG("Got unexpected info: ~p", [Info]), + {noreply, State}. + +-spec terminate(normal | shutdown | {shutdown, term()} | term(), state()) -> ok. +terminate(Reason, + #state{host = Host, + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode, + rtbl_retry_timer = RTBLRetryTimer} = State) -> + ?DEBUG("Stopping spam filter process for ~s: ~p", [Host, Reason]), + misc:cancel_timer(RTBLRetryTimer), + DumpFile = gen_mod:get_module_opt(Host, ?MODULE, spam_dump_file), + DumpFile1 = expand_host(DumpFile, Host), + close_dump_file(DumpFile1, State), + ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, + s2s_receive_packet, 50), + ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, + sm_receive_packet, 50), + ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, + s2s_in_handle_info, 90), + ejabberd_hooks:delete(local_send_to_resource_hook, Host, + mod_antispam_rtbl, pubsub_event_handler, 50), + mod_antispam_rtbl:unsubscribe(RTBLHost, RTBLDomainsNode,Host), + case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of + false -> + ejabberd_hooks:delete(reopen_log_hook, ?MODULE, + reopen_log, 50); + true -> + ok + end. + +-spec code_change({down, term()} | term(), state(), term()) -> {ok, state()}. +code_change(_OldVsn, #state{host = Host} = State, _Extra) -> + ?DEBUG("Updating spam filter process for ~s", [Host]), + {ok, State}. + +%%-------------------------------------------------------------------- +%% Hook callbacks. +%%-------------------------------------------------------------------- + +-spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) + -> {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}. +s2s_receive_packet({A, State}) -> + case sm_receive_packet(A) of + {stop, drop} -> + {stop, {drop, State}}; + Result -> + {Result, State} + end. + +-spec sm_receive_packet(stanza() | drop) -> stanza() | drop | {stop, drop}. +sm_receive_packet(drop = Acc) -> + Acc; +sm_receive_packet(#message{from = From, + to = #jid{lserver = LServer} = To, + type = Type} = Msg) + when Type /= groupchat, + Type /= error -> + do_check(From, To, LServer, Msg); +sm_receive_packet(#presence{from = From, + to = #jid{lserver = LServer} = To, + type = subscribe} = Presence) -> + do_check(From, To, LServer, Presence); +sm_receive_packet(Acc) -> + Acc. + +do_check(From, To, LServer, Stanza) -> + case needs_checking(From, To) of + true -> + case check_from(LServer, From) of + ham -> + case check_stanza(LServer, From, Stanza) of + ham -> + Stanza; + spam -> + reject(Stanza), + {stop, drop} + end; + spam -> + reject(Stanza), + {stop, drop} + end; + false -> + Stanza + end. + +check_stanza(LServer, From, #message{body = Body}) -> + check_body(LServer, From, xmpp:get_text(Body)); +check_stanza(_, _, _) -> + ham. + +-spec s2s_in_handle_info(s2s_in_state(), any()) + -> s2s_in_state() | {stop, s2s_in_state()}. +s2s_in_handle_info(State, {_Ref, {spam_filter, _}}) -> + ?DEBUG("Dropping expired spam filter result", []), + {stop, State}; +s2s_in_handle_info(State, _) -> + State. + +-spec reopen_log() -> ok. +reopen_log() -> + lists:foreach(fun(Host) -> + Proc = get_proc_name(Host), + gen_server:cast(Proc, reopen_log) + end, get_spam_filter_hosts()). + +%%-------------------------------------------------------------------- +%% Internal functions. +%%-------------------------------------------------------------------- +-spec needs_checking(jid(), jid()) -> boolean(). +needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) -> + case gen_mod:is_loaded(LServer, ?MODULE) of + true -> + Access = gen_mod:get_module_opt(LServer, ?MODULE, access_spam), + case acl:match_rule(LServer, Access, To) of + allow -> + ?DEBUG("Spam not filtered for ~s", [jid:encode(To)]), + false; + deny -> + ?DEBUG("Spam is filtered for ~s", [jid:encode(To)]), + not mod_roster:is_subscribed(From, To) andalso + not mod_roster:is_subscribed(jid:make(<<>>, FromHost), To) % likely a gateway + end; + false -> + ?DEBUG("~s not loaded for ~s", [?MODULE, LServer]), + false + end. + +-spec check_from(binary(), jid()) -> ham | spam. +check_from(Host, From) -> + Proc = get_proc_name(Host), + LFrom = {_, FromDomain, _} = jid:remove_resource(jid:tolower(From)), + try + case gen_server:call(Proc, {is_blocked_domain, FromDomain}) of + true -> + ?DEBUG("Spam JID found in blocked domains: ~p", [From]), + ejabberd_hooks:run(spam_found, Host, [{jid, From}]), + spam; + false -> + case gen_server:call(Proc, {check_jid, LFrom}) of + {spam_filter, Result} -> + Result + end + end + catch exit:{timeout, _} -> + ?WARNING_MSG("Timeout while checking ~s against list of blocked domains or spammers", + [jid:encode(From)]), + ham + end. + +-spec check_body(binary(), jid(), binary()) -> ham | spam. +check_body(Host, From, Body) -> + case {extract_urls(Host, Body), extract_jids(Body)} of + {none, none} -> + ?DEBUG("No JIDs/URLs found in message", []), + ham; + {URLs, JIDs} -> + Proc = get_proc_name(Host), + LFrom = jid:remove_resource(jid:tolower(From)), + try gen_server:call(Proc, {check_body, URLs, JIDs, LFrom}) of + {spam_filter, Result} -> + Result + catch exit:{timeout, _} -> + ?WARNING_MSG("Timeout while checking body", []), + ham + end + end. + +-spec extract_urls(binary(), binary()) -> {urls, [url()]} | none. +extract_urls(Host, Body) -> + RE = <<"https?://\\S+">>, + Options = [global, {capture, all, binary}], + case re:run(Body, RE, Options) of + {match, Captured} when is_list(Captured) -> + Urls = resolve_redirects(Host, lists:flatten(Captured)), + {urls, Urls}; + nomatch -> + none + end. + +-spec resolve_redirects(binary(), [url()]) -> [url()]. +resolve_redirects(Host, URLs) -> + Proc = get_proc_name(Host), + try gen_server:call(Proc, {resolve_redirects, URLs}) of + {spam_filter, ResolvedURLs} -> + ResolvedURLs + catch exit:{timeout, _} -> + ?WARNING_MSG("Timeout while resolving redirects: ~p", [URLs]), + URLs + end. + +-spec do_resolve_redirects([url()], [url()]) -> [url()]. +do_resolve_redirects([], Result) -> Result; +do_resolve_redirects([URL | Rest], Acc) -> + case + httpc:request(get, {URL, [{"user-agent", "curl/8.7.1"}]}, + [{autoredirect, false}, {timeout, ?HTTPC_TIMEOUT}], []) + of + {ok, {{_, StatusCode, _}, Headers, _Body}} when StatusCode >= 300, StatusCode < 400 -> + Location = proplists:get_value("location", Headers), + case Location == undefined orelse lists:member(Location, Acc) of + true -> + do_resolve_redirects(Rest, [URL | Acc]); + false -> + do_resolve_redirects([Location | Rest], [URL | Acc]) + end; + _Res -> + do_resolve_redirects(Rest, [URL | Acc]) + end. + +-spec extract_jids(binary()) -> {jids, [ljid()]} | none. +extract_jids(Body) -> + RE = <<"\\S+@\\S+">>, + Options = [global, {capture, all, binary}], + case re:run(Body, RE, Options) of + {match, Captured} when is_list(Captured) -> + {jids, lists:filtermap(fun try_decode_jid/1, + lists:flatten(Captured))}; + nomatch -> + none + end. + +-spec try_decode_jid(binary()) -> {true, ljid()} | false. +try_decode_jid(S) -> + try jid:decode(S) of + #jid{} = JID -> + {true, jid:remove_resource(jid:tolower(JID))} + catch _:{bad_jid, _} -> + false + end. + +-spec filter_jid(ljid(), jid_set(), state()) -> {ham | spam, state()}. +filter_jid(From, Set, #state{host = Host} = State) -> + case sets:is_element(From, Set) of + true -> + ?DEBUG("Spam JID found: ~s", [jid:encode(From)]), + ejabberd_hooks:run(spam_found, Host, [{jid, From}]), + {spam, State}; + false -> + case cache_lookup(From, State) of + {true, State1} -> + ?DEBUG("Spam JID found: ~s", [jid:encode(From)]), + ejabberd_hooks:run(spam_found, Host, [{jid, From}]), + {spam, State1}; + {false, State1} -> + ?DEBUG("JID not listed: ~s", [jid:encode(From)]), + {ham, State1} + end + end. + +-spec filter_body({urls, [url()]} | {jids, [ljid()]} | none, + url_set() | jid_set(), jid(), state()) + -> {ham | spam, state()}. +filter_body({_, Addrs}, Set, From, #state{host = Host} = State) -> + case lists:any(fun(Addr) -> sets:is_element(Addr, Set) end, Addrs) of + true -> + ?DEBUG("Spam addresses found: ~p", [Addrs]), + ejabberd_hooks:run(spam_found, Host, [{body, Addrs}]), + {spam, cache_insert(From, State)}; + false -> + ?DEBUG("Addresses not listed: ~p", [Addrs]), + {ham, State} + end; +filter_body(none, _Set, _From, State) -> + {ham, State}. + +-spec reload_files(#{Type :: atom() => filename()}, state()) + -> {ok | {error, binary()}, state()}. +reload_files(Files, #state{host = Host, blocked_domains = BlockedDomains} = State) -> + try read_files(Files) of + #{jid := JIDsSet, url := URLsSet, domains := SpamDomainsSet, whitelist_domains := WhitelistDomains} -> + case sets_equal(JIDsSet, State#state.jid_set) of + true -> + ?INFO_MSG("Reloaded spam JIDs for ~s (unchanged)", [Host]); + false -> + ?INFO_MSG("Reloaded spam JIDs for ~s (changed)", [Host]) + end, + case sets_equal(URLsSet, State#state.url_set) of + true -> + ?INFO_MSG("Reloaded spam URLs for ~s (unchanged)", [Host]); + false -> + ?INFO_MSG("Reloaded spam URLs for ~s (changed)", [Host]) + end, + {ok, State#state{jid_set = JIDsSet, + url_set = URLsSet, + blocked_domains = maps:merge(BlockedDomains, set_to_map(SpamDomainsSet)), + whitelist_domains = set_to_map(WhitelistDomains, false)} + } + catch {Op, File, Reason} when Op == open; + Op == read -> + Txt = format("Cannot ~s ~s for ~s: ~s", + [Op, File, Host, format_error(Reason)]), + ?ERROR_MSG("~s", [Txt]), + {{error, Txt}, State} + end. + +set_to_map(Set) -> + set_to_map(Set, true). + +set_to_map(Set, V) -> + sets:fold(fun(K, M) -> M#{K => V} end, #{}, Set). + +-spec read_files(#{Type => filename()}) -> #{jid => jid_set(), url => url_set(), Type => sets:set(binary())} + when Type :: atom(). +read_files(Files) -> + maps:map(fun(Type, Filename) -> + read_file(Filename, line_parser(Type)) + end, + Files). + +-spec line_parser(Type :: atom()) -> fun((binary()) -> binary()). +line_parser(jid) -> fun parse_jid/1; +line_parser(url) -> fun parse_url/1; +line_parser(_) -> fun trim/1. + +-spec read_file(filename(), fun((binary()) -> ljid() | url())) + -> jid_set() | url_set(). +read_file(none, _ParseLine) -> + sets:new(); +read_file(File, ParseLine) -> + case file:open(File, [read, binary, raw, {read_ahead, 65536}]) of + {ok, Fd} -> + try read_line(Fd, ParseLine, sets:new()) + catch throw:E -> throw({read, File, E}) + after ok = file:close(Fd) + end; + {error, Reason} -> + throw({open, File, Reason}) + end. + +-spec read_line(file:io_device(), fun((binary()) -> ljid() | url()), + jid_set() | url_set()) + -> jid_set() | url_set(). +read_line(Fd, ParseLine, Set) -> + case file:read_line(Fd) of + {ok, Line} -> + read_line(Fd, ParseLine, sets:add_element(ParseLine(Line), Set)); + {error, Reason} -> + throw(Reason); + eof -> + Set + end. + +-spec parse_jid(binary()) -> ljid(). +parse_jid(S) -> + try jid:decode(trim(S)) of + #jid{} = JID -> + jid:remove_resource(jid:tolower(JID)) + catch _:{bad_jid, _} -> + throw({bad_jid, S}) + end. + +-spec parse_url(binary()) -> url(). +parse_url(S) -> + URL = trim(S), + RE = <<"https?://\\S+$">>, + Options = [anchored, caseless, {capture, none}], + case re:run(URL, RE, Options) of + match -> + URL; + nomatch -> + throw({bad_url, S}) + end. + +-spec trim(binary()) -> binary(). +trim(S) -> + re:replace(S, <<"\\s+$">>, <<>>, [{return, binary}]). + +-spec reject(stanza()) -> ok. +reject(#message{from = From, to = To, type = Type, lang = Lang} = Msg) + when Type /= groupchat, + Type /= error -> + ?INFO_MSG("Rejecting unsolicited message from ~s to ~s", + [jid:encode(From), jid:encode(To)]), + Txt = <<"Your message is unsolicited">>, + Err = xmpp:err_policy_violation(Txt, Lang), + maybe_dump_spam(Msg), + ejabberd_router:route_error(Msg, Err); +reject(#presence{from = From, to = To, lang = Lang} = Presence) -> + ?INFO_MSG("Rejecting unsolicited presence from ~s to ~s", + [jid:encode(From), jid:encode(To)]), + Txt = <<"Your traffic is unsolicited">>, + Err = xmpp:err_policy_violation(Txt, Lang), + ejabberd_router:route_error(Presence, Err); +reject(_) -> + ok. + +-spec open_dump_file(filename(), state()) -> state(). +open_dump_file(none, State) -> + State#state{dump_fd = undefined}; +open_dump_file(Name, State) -> + Modes = [append, raw, binary, delayed_write], + case file:open(Name, Modes) of + {ok, Fd} -> + ?DEBUG("Opened ~s", [Name]), + State#state{dump_fd = Fd}; + {error, Reason} -> + ?ERROR_MSG("Cannot open dump file ~s: ~s", [Name, file:format_error(Reason)]), + State#state{dump_fd = undefined} + end. + +-spec close_dump_file(filename(), state()) -> ok. +close_dump_file(_Name, #state{dump_fd = undefined}) -> + ok; +close_dump_file(Name, #state{dump_fd = Fd}) -> + case file:close(Fd) of + ok -> + ?DEBUG("Closed ~s", [Name]); + {error, Reason} -> + ?ERROR_MSG("Cannot close ~s: ~s", [Name, file:format_error(Reason)]) + end. + +-spec reopen_dump_file(state()) -> state(). +reopen_dump_file(#state{host = Host} = State) -> + DumpFile = gen_mod:get_module_opt(Host, ?MODULE, spam_dump_file), + DumpFile1 = expand_host(DumpFile, Host), + close_dump_file(DumpFile1, State), + open_dump_file(DumpFile1, State). + +-spec maybe_dump_spam(message()) -> ok. +maybe_dump_spam(#message{to = #jid{lserver = LServer}} = Msg) -> + By = jid:make(<<>>, LServer), + Proc = get_proc_name(LServer), + Time = erlang:timestamp(), + Msg1 = misc:add_delay_info(Msg, By, Time), + XML = fxml:element_to_binary(xmpp:encode(Msg1)), + gen_server:cast(Proc, {dump, XML}). + +-spec get_proc_name(binary()) -> atom(). +get_proc_name(Host) -> + gen_mod:get_module_proc(Host, ?MODULE). + +-spec get_spam_filter_hosts() -> [binary()]. +get_spam_filter_hosts() -> + [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, ?MODULE)]. + +-spec expand_host(binary() | none, binary()) -> binary() | none. +expand_host(none, _Host) -> + none; +expand_host(Input, Host) -> + misc:expand_keyword(<<"@HOST@">>, Input, Host). + +-spec sets_equal(sets:set(), sets:set()) -> boolean(). +sets_equal(A, B) -> + sets:is_subset(A, B) andalso sets:is_subset(B, A). + +-spec format(io:format(), [term()]) -> binary(). +format(Format, Data) -> + iolist_to_binary(io_lib:format(Format, Data)). + +-spec format_error(atom() | tuple()) -> binary(). +format_error({bad_jid, JID}) -> + <<"Not a valid JID: ", JID/binary>>; +format_error({bad_url, URL}) -> + <<"Not an HTTP(S) URL: ", URL/binary>>; +format_error(Reason) -> + list_to_binary(file:format_error(Reason)). + +%%-------------------------------------------------------------------- +%% Caching. +%%-------------------------------------------------------------------- +-spec cache_insert(ljid(), state()) -> state(). +cache_insert(_LJID, #state{max_cache_size = 0} = State) -> + State; +cache_insert(LJID, #state{jid_cache = Cache, max_cache_size = MaxSize} = State) + when MaxSize /= unlimited, map_size(Cache) >= MaxSize -> + cache_insert(LJID, shrink_cache(State)); +cache_insert(LJID, #state{jid_cache = Cache} = State) -> + ?INFO_MSG("Caching spam JID: ~s", [jid:encode(LJID)]), + Cache1 = Cache#{LJID => erlang:monotonic_time(second)}, + State#state{jid_cache = Cache1}. + +-spec cache_lookup(ljid(), state()) -> {boolean(), state()}. +cache_lookup(LJID, #state{jid_cache = Cache} = State) -> + case Cache of + #{LJID := _Timestamp} -> + Cache1 = Cache#{LJID => erlang:monotonic_time(second)}, + State1 = State#state{jid_cache = Cache1}, + {true, State1}; + #{} -> + {false, State} + end. + +-spec shrink_cache(state()) -> state(). +shrink_cache(#state{jid_cache = Cache, max_cache_size = MaxSize} = State) -> + ShrinkedSize = round(MaxSize / 2), + N = map_size(Cache) - ShrinkedSize, + L = lists:keysort(2, maps:to_list(Cache)), + Cache1 = maps:from_list(lists:nthtail(N, L)), + State#state{jid_cache = Cache1}. + +-spec expire_cache(integer(), state()) -> {{ok, binary()}, state()}. +expire_cache(Age, #state{jid_cache = Cache} = State) -> + Threshold = erlang:monotonic_time(second) - Age, + Cache1 = maps:filter(fun(_, TS) -> TS >= Threshold end, Cache), + NumExp = map_size(Cache) - map_size(Cache1), + Txt = format("Expired ~B cache entries", [NumExp]), + {{ok, Txt}, State#state{jid_cache = Cache1}}. + +-spec add_to_cache(ljid(), state()) -> {{ok, binary()}, state()}. +add_to_cache(LJID, State) -> + State1 = cache_insert(LJID, State), + Txt = format("~s added to cache", [jid:encode(LJID)]), + {{ok, Txt}, State1}. + +-spec drop_from_cache(ljid(), state()) -> {{ok, binary()}, state()}. +drop_from_cache(LJID, #state{jid_cache = Cache} = State) -> + Cache1 = maps:remove(LJID, Cache), + if map_size(Cache1) < map_size(Cache) -> + Txt = format("~s removed from cache", [jid:encode(LJID)]), + {{ok, Txt}, State#state{jid_cache = Cache1}}; + true -> + Txt = format("~s wasn't cached", [jid:encode(LJID)]), + {{ok, Txt}, State} + end. + +%%-------------------------------------------------------------------- +%% ejabberd command callbacks. +%%-------------------------------------------------------------------- +-spec get_commands_spec() -> [ejabberd_commands()]. +get_commands_spec() -> + [#ejabberd_commands{name = reload_spam_filter_files, tags = [filter], + desc = "Reload spam JID/URL files", + module = ?MODULE, function = reload_spam_filter_files, + args = [{host, binary}], + result = {res, rescode}}, + #ejabberd_commands{name = get_spam_filter_cache, tags = [filter], + desc = "Show spam filter cache contents", + module = ?MODULE, function = get_spam_filter_cache, + args = [{host, binary}], + result = {spammers, {list, {spammer, {tuple, + [{jid, string}, {timestamp, integer}]}}}}}, + #ejabberd_commands{name = expire_spam_filter_cache, tags = [filter], + desc = "Remove old/unused spam JIDs from cache", + module = ?MODULE, function = expire_spam_filter_cache, + args = [{host, binary}, {seconds, integer}], + result = {res, restuple}}, + #ejabberd_commands{name = add_to_spam_filter_cache, tags = [filter], + desc = "Add JID to spam filter cache", + module = ?MODULE, + function = add_to_spam_filter_cache, + args = [{host, binary}, {jid, binary}], + result = {res, restuple}}, + #ejabberd_commands{name = drop_from_spam_filter_cache, tags = [filter], + desc = "Drop JID from spam filter cache", + module = ?MODULE, + function = drop_from_spam_filter_cache, + args = [{host, binary}, {jid, binary}], + result = {res, restuple}}, + #ejabberd_commands{name = get_blocked_domains, tags = [filter], + desc = "Get list of domains being blocked", + module = ?MODULE, + function = get_blocked_domains, + args = [{host, binary}], + result = {blocked_domains, {list, {jid, string}}}}, + #ejabberd_commands{name = add_blocked_domain, tags = [filter], + desc = "Add domain to list of blocked domains", + module = ?MODULE, + function = add_blocked_domain, + args = [{host, binary}, {domain, binary}], + result = {res, restuple}}, + #ejabberd_commands{name = remove_blocked_domain, tags = [filter], + desc = "Remove domain from list of blocked domains", + module = ?MODULE, + function = remove_blocked_domain, + args = [{host, binary}, {domain, binary}], + result = {res, restuple}} + ]. + +for_all_hosts(F, A) -> + try lists:map( + fun(Host) -> + apply(F, [Host | A]) + end, get_spam_filter_hosts()) of + List -> + case lists:filter(fun({error, _}) -> true; (_) -> false end, List) of + [] -> hd(List); + Errors -> hd(Errors) + end + catch error:{badmatch, {error, _Reason} = Error} -> + Error + end. + +try_call_by_host(Host, Call) -> + LServer = jid:nameprep(Host), + Proc = get_proc_name(LServer), + try gen_server:call(Proc, Call, ?COMMAND_TIMEOUT) of + Result -> + Result + catch exit:{noproc, _} -> + {error, "Not configured for " ++ binary_to_list(Host)}; + exit:{timeout, _} -> + {error, "Timeout while querying ejabberd"} + end. + +-spec reload_spam_filter_files(binary()) -> ok | {error, string()}. +reload_spam_filter_files(<<"global">>) -> + for_all_hosts(fun reload_spam_filter_files/1, []); +reload_spam_filter_files(Host) -> + LServer = jid:nameprep(Host), + Files = #{domains => gen_mod:get_module_opt(LServer, ?MODULE, spam_domains_file), + jid => gen_mod:get_module_opt(LServer, ?MODULE, spam_jids_file), + url => gen_mod:get_module_opt(LServer, ?MODULE, spam_urls_file)}, + case try_call_by_host(Host, {reload_files, Files}) of + {spam_filter, ok} -> + ok; + {spam_filter, {error, Txt}} -> + {error, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end. + +-spec get_blocked_domains(binary()) -> [binary()]. +get_blocked_domains(Host) -> + case try_call_by_host(Host, get_blocked_domains) of + {blocked_domains, BlockedDomains} -> + maps:keys(maps:filter(fun(_, false) -> false; (_, _) -> true end, BlockedDomains)); + {error, _R} = Error -> + Error + end. + +-spec add_blocked_domain(binary(), binary()) -> {ok, string()}. +add_blocked_domain(<<"global">>, Domain) -> + for_all_hosts(fun add_blocked_domain/2, [Domain]); +add_blocked_domain(Host, Domain) -> + case try_call_by_host(Host, {add_blocked_domain, Domain}) of + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end. + +-spec remove_blocked_domain(binary(), binary()) -> {ok, string()}. +remove_blocked_domain(<<"global">>, Domain) -> + for_all_hosts(fun remove_blocked_domain/2, [Domain]); +remove_blocked_domain(Host, Domain) -> + case try_call_by_host(Host, {remove_blocked_domain, Domain}) of + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end. + +-spec get_spam_filter_cache(binary()) + -> [{binary(), integer()}] | {error, string()}. +get_spam_filter_cache(Host) -> + case try_call_by_host(Host, get_cache) of + {spam_filter, Cache} -> + [{jid:encode(JID), TS + erlang:time_offset(second)} || + {JID, TS} <- Cache]; + {error, _R} = Error -> + Error + end. + +-spec expire_spam_filter_cache(binary(), integer()) -> {ok | error, string()}. +expire_spam_filter_cache(<<"global">>, Age) -> + for_all_hosts(fun expire_spam_filter_cache/2, [Age]); +expire_spam_filter_cache(Host, Age) -> + case try_call_by_host(Host, {expire_cache, Age}) of + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end. + +-spec add_to_spam_filter_cache(binary(), binary()) -> [{binary(), integer()}] | {error, string()}. +add_to_spam_filter_cache(<<"global">>, JID) -> + for_all_hosts(fun add_to_spam_filter_cache/2, [JID]); +add_to_spam_filter_cache(Host, EncJID) -> + try jid:decode(EncJID) of + #jid{} = JID -> + LJID = jid:remove_resource(jid:tolower(JID)), + case try_call_by_host(Host, {add_to_cache, LJID}) of + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end + catch _:{bad_jid, _} -> + {error, "Not a valid JID: " ++ binary_to_list(EncJID)} + end. + +-spec drop_from_spam_filter_cache(binary(), binary()) -> {ok | error, string()}. +drop_from_spam_filter_cache(<<"global">>, JID) -> + for_all_hosts(fun drop_from_spam_filter_cache/2, [JID]); +drop_from_spam_filter_cache(Host, EncJID) -> + try jid:decode(EncJID) of + #jid{} = JID -> + LJID = jid:remove_resource(jid:tolower(JID)), + case try_call_by_host(Host, {drop_from_cache, LJID}) of + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end + catch _:{bad_jid, _} -> + {error, "Not a valid JID: " ++ binary_to_list(EncJID)} + end. diff --git a/src/mod_antispam_rtbl.erl b/src/mod_antispam_rtbl.erl new file mode 100644 index 000000000..f9994fd08 --- /dev/null +++ b/src/mod_antispam_rtbl.erl @@ -0,0 +1,132 @@ +%%%---------------------------------------------------------------------- +%%% File : mod_antispam_rtbl.erl +%%% Author : Stefan Strigler +%%% Purpose : Collection of RTBL specific functionality +%%% Created : 20 Mar 2025 by Stefan Strigler +%%% +%%% +%%% ejabberd, Copyright (C) 2025 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%---------------------------------------------------------------------- +-module(mod_antispam_rtbl). +-author('stefan@strigler.de'). + +-include_lib("xmpp/include/xmpp.hrl"). +-include("logger.hrl"). + +-define(SERVICE_MODULE, mod_antispam). +-define(SERVICE_JID_PREFIX, "rtbl-"). + +-export([parse_blocked_domains/1, + parse_pubsub_event/1, + pubsub_event_handler/1, + request_blocked_domains/3, + subscribe/3, + unsubscribe/3]). + +subscribe(RTBLHost, RTBLDomainsNode, From) -> + FromJID = service_jid(From), + SubIQ = #iq{type = set, to = jid:make(RTBLHost), from = FromJID, + sub_els = [ + #pubsub{subscribe = #ps_subscribe{jid = FromJID, node = RTBLDomainsNode}}]}, + ?DEBUG("Sending subscription request:~n~p", [xmpp:encode(SubIQ)]), + ejabberd_router:route_iq(SubIQ, subscribe_result, self()). + +-spec unsubscribe(binary() | none, binary(), binary()) -> ok. +unsubscribe(none, _PSNode, _From) -> + ok; +unsubscribe(RTBLHost, RTBLDomainsNode, From) -> + FromJID = jid:make(From), + SubIQ = #iq{type = set, to = jid:make(RTBLHost), from = FromJID, + sub_els = [ + #pubsub{unsubscribe = #ps_unsubscribe{jid = FromJID, node = RTBLDomainsNode}}]}, + ejabberd_router:route_iq(SubIQ, unsubscribe_result, self()). + +-spec request_blocked_domains(binary() | none, binary(), binary()) -> ok. +request_blocked_domains(none, _PSNode, _From) -> + ok; +request_blocked_domains(RTBLHost, RTBLDomainsNode, From) -> + IQ = #iq{type = get, from = jid:make(From), + to = jid:make(RTBLHost), + sub_els = [ + #pubsub{items = #ps_items{node = RTBLDomainsNode}}]}, + ?DEBUG("Requesting RTBL blocked domains from ~s:~n~p", [RTBLHost, xmpp:encode(IQ)]), + ejabberd_router:route_iq(IQ, blocked_domains, self()). + +-spec parse_blocked_domains(stanza()) -> #{binary() => any()} | undefined. +parse_blocked_domains(#iq{to = #jid{lserver = LServer}, type = result} = IQ) -> + ?DEBUG("parsing iq-result items: ~p", [IQ]), + RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), + case xmpp:get_subtag(IQ, #pubsub{}) of + #pubsub{items = #ps_items{node = RTBLDomainsNode, items = Items}} -> + ?DEBUG("Got items:~n~p", [Items]), + parse_items(Items); + _ -> + undefined + end. + +-spec parse_pubsub_event(stanza()) -> #{binary() => any()}. +parse_pubsub_event(#message{to = #jid{lserver = LServer}} = Msg) -> + RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), + case xmpp:get_subtag(Msg, #ps_event{}) of + #ps_event{items = #ps_items{node = RTBLDomainsNode, retract = ID}} when ID /= undefined -> + ?DEBUG("Got item to retract:~n~p", [ID]), + #{ID => false}; + #ps_event{items = #ps_items{node = RTBLDomainsNode, items = Items}} -> + ?DEBUG("Got items:~n~p", [Items]), + parse_items(Items); + Other -> + ?WARNING_MSG("Couldn't extract items: ~p", [Other]), + #{} + end. + +-spec parse_items([ps_item()]) -> #{binary() => any()}. +parse_items(Items) -> + lists:foldl( + fun(#ps_item{id = ID}, Acc) -> + %% TODO extract meta/extra instructions + maps:put(ID, true, Acc) + end, #{}, Items). + +-spec service_jid(binary()) -> jid(). +service_jid(Host) -> + jid:make(<<>>, Host, <>). + +%%-------------------------------------------------------------------- +%% Hook callbacks. +%%-------------------------------------------------------------------- + +-spec pubsub_event_handler(stanza()) -> drop | stanza(). +pubsub_event_handler(#message{from = FromJid, + to = #jid{lserver = LServer, + lresource = <>}} = Msg) -> + ?DEBUG("Got RTBL message:~n~p", [Msg]), + From = jid:encode(FromJid), + case gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_host) of + From -> + ParsedItems = parse_pubsub_event(Msg), + Proc = gen_mod:get_module_proc(LServer, ?SERVICE_MODULE), + gen_server:cast(Proc, {update_blocked_domains, ParsedItems}), + %% FIXME what's the difference between `{drop, ...}` and `{stop, {drop, ...}}`? + drop; + _Other -> + ?INFO_MSG("Got unexpected message from ~s to rtbl resource:~n~p", [From, Msg]), + Msg + end; +pubsub_event_handler(Acc) -> + ?DEBUG("unexpected something on pubsub_event_handler: ~p", [Acc]), + Acc. From 70bec7b714351cdb643811907f86a7fd1f76106a Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 22 May 2025 18:27:13 +0200 Subject: [PATCH 02/27] tests: update readme and compose to work with current sw versions --- test/docker/README.md | 8 ++++---- test/docker/docker-compose.yml | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/test/docker/README.md b/test/docker/README.md index 93b1a3504..7caaa4bb7 100644 --- a/test/docker/README.md +++ b/test/docker/README.md @@ -10,7 +10,7 @@ attached to it. ``` mkdir test/docker/db/mysql/data mkdir test/docker/db/postgres/data -(cd test/docker; docker-compose up) +(cd test/docker; docker compose up) ``` You can stop all the databases with CTRL-C. @@ -20,8 +20,8 @@ You can stop all the databases with CTRL-C. The following commands will create the necessary login, user and database, will grant rights on the database in MSSQL and create the ejabberd schema: ``` -docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -i /initdb_mssql.sql -docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -d ejabberd_test -i /mssql.sql +docker exec ejabberd-mssql /opt/mssql-tools18/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -i /initdb_mssql.sql -C +docker exec ejabberd-mssql /opt/mssql-tools18/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -d ejabberd_test -i /mssql.sql -C ``` ## Running tests @@ -44,7 +44,7 @@ make test You can fully clean up the environment with: ``` -(cd test/docker; docker-compose down) +(cd test/docker; docker compose down) ``` If you want to clean the data, you can remove the data volumes after the `docker-compose down` command: diff --git a/test/docker/docker-compose.yml b/test/docker/docker-compose.yml index bf99ae04e..bdc470a63 100644 --- a/test/docker/docker-compose.yml +++ b/test/docker/docker-compose.yml @@ -7,7 +7,6 @@ services: volumes: - mysqldata:/var/lib/mysql - ../../sql/mysql.sql:/docker-entrypoint-initdb.d/mysql.sql:ro - command: --default-authentication-plugin=mysql_native_password restart: always ports: - 3306:3306 From c48aa38c39e909eaaeb790e1fec50463aaf857d3 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 22 May 2025 18:26:12 +0200 Subject: [PATCH 03/27] mod_antispam: add test suite --- test/antispam_tests.erl | 152 +++++++++++++++++++ test/ejabberd_SUITE.erl | 5 + test/ejabberd_SUITE_data/ejabberd.mnesia.yml | 5 + test/ejabberd_SUITE_data/ejabberd.redis.yml | 5 + test/ejabberd_SUITE_data/spam_domains.txt | 1 + test/ejabberd_SUITE_data/spam_jids.txt | 1 + test/ejabberd_SUITE_data/spam_urls.txt | 1 + test/suite.erl | 23 +++ test/suite.hrl | 2 + 9 files changed, 195 insertions(+) create mode 100644 test/antispam_tests.erl create mode 100644 test/ejabberd_SUITE_data/spam_domains.txt create mode 100644 test/ejabberd_SUITE_data/spam_jids.txt create mode 100644 test/ejabberd_SUITE_data/spam_urls.txt diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl new file mode 100644 index 000000000..d7ce196c0 --- /dev/null +++ b/test/antispam_tests.erl @@ -0,0 +1,152 @@ +%%%------------------------------------------------------------------- +%%% Author : Stefan Strigler +%%% Created : 8 May 2025 by Stefan Strigler +%%% +%%% +%%% ejabberd, Copyright (C) 2025 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%---------------------------------------------------------------------- +-module(antispam_tests). + +-compile(export_all). + +-import(suite, [recv_presence/1, send_recv/2, my_jid/1, muc_room_jid/1, + send/2, recv_message/1, recv_iq/1, muc_jid/1, + alt_room_jid/1, wait_for_slave/1, wait_for_master/1, + disconnect/1, put_event/2, get_event/1, peer_muc_jid/1, + my_muc_jid/1, get_features/2, set_opt/3]). +-include("suite.hrl"). + +%%%=================================================================== +%%% API +%%%=================================================================== +%%%=================================================================== +%%% Single tests +%%%=================================================================== +single_cases() -> + {antispam_single, [sequence], + [single_test(spam_files), + single_test(blocked_domains), + single_test(jid_cache), + single_test(rtbl_domains)]}. + +spam_files(Config) -> + Host = ?config(server, Config), + To = my_jid(Config), + + SpamJID = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>), + SpamJIDMsg = #message{from = SpamJID, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + is_spam(SpamJIDMsg), + + Spammer = jid:make(<<"spammer">>, <<"localhost">>, <<"spam_client">>), + NoSpamMsg = #message{from = Spammer, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + is_not_spam(NoSpamMsg), + SpamMsg = #message{from = Spammer, to = To, type = chat, body = [#text{data = <<"hello world\nhttps://spam.domain.url">>}]}, + is_spam(SpamMsg), + %% now check this mischief is in jid_cache + is_spam(NoSpamMsg), + mod_antispam:drop_from_spam_filter_cache(Host, jid:to_string(Spammer)), + is_not_spam(NoSpamMsg), + + ?retry(100, 10, + ?match(true, (has_spam_domain(<<"spam_domain.org">>))(Host))), + + SpamDomain = jid:make(<<"spammer">>, <<"spam_domain.org">>, <<"spam_client">>), + SpamDomainMsg = #message{from = SpamDomain, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + is_spam(SpamDomainMsg), + ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)), + ?match([], mod_antispam:get_blocked_domains(Host)), + is_not_spam(SpamDomainMsg), + disconnect(Config). + +blocked_domains(Config) -> + Host = ?config(server, Config), + ?match([], mod_antispam:get_blocked_domains(Host)), + SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), + To = my_jid(Config), + Msg = #message{from = SpamFrom, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + is_not_spam(Msg), + ?match({ok, _}, mod_antispam:add_blocked_domain(<<"global">>, <<"spam.domain">>)), + is_spam(Msg), + Vhosts = [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, mod_antispam)], + NumVhosts = length(Vhosts), + ?match(NumVhosts, length(lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts))), + ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam.domain">>)), + ?match([], mod_antispam:get_blocked_domains(Host)), + is_not_spam(Msg), + ?match(NumVhosts, length(lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts)) + 1), + ?match({ok, _}, mod_antispam:remove_blocked_domain(<<"global">>, <<"spam.domain">>)), + ?match([], lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts)), + ?match({ok, _}, mod_antispam:add_blocked_domain(Host, <<"spam.domain">>)), + ?match([Host], lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts)), + is_spam(Msg), + ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam.domain">>)), + is_not_spam(Msg), + disconnect(Config). + +jid_cache(Config) -> + Host = ?config(server, Config), + SpamFrom = jid:make(<<"spammer">>, Host, <<"spam_client">>), + To = my_jid(Config), + Msg = #message{from = SpamFrom, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + is_not_spam(Msg), + mod_antispam:add_to_spam_filter_cache(Host, jid:to_string(SpamFrom)), + is_spam(Msg), + mod_antispam:drop_from_spam_filter_cache(Host, jid:to_string(SpamFrom)), + is_not_spam(Msg), + disconnect(Config). + +rtbl_domains(Config) -> + Host = ?config(server, Config), + RTBLHost = jid:to_string(suite:pubsub_jid(Config)), + RTBLDomainsNode = <<"spam_source_domains">>, + OldOpts = gen_mod:get_module_opts(Host, mod_antispam), + NewOpts = maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}), + Owner = jid:make(?config(user, Config), ?config(server, Config), <<>>), + {result, _} = mod_pubsub:create_node(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"flat">>), + {result, _} = mod_pubsub:publish_item(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"spam.source.domain">>, + [xmpp:encode(#ps_item{id = <<"spam.source.domain">>, sub_els = []})]), + mod_antispam:reload(Host, OldOpts, NewOpts), + ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)), + ?retry(100, 10, + ?match([<<"spam.source.domain">>], mod_antispam:get_blocked_domains(Host))), + {result, _} = mod_pubsub:publish_item(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"spam.source.another">>, + [xmpp:encode(#ps_item{id = <<"spam.source.another">>, sub_els = []})]), + ?retry(100, 10, + ?match(true, (has_spam_domain(<<"spam.source.another">>))(Host))), + {result, _} = mod_pubsub:delete_item(RTBLHost, RTBLDomainsNode, Owner, <<"spam.source.another">>, true), + ?retry(100, 10, + ?match(false, (has_spam_domain(<<"spam.source.another">>))(Host))), + {result, _} = mod_pubsub:delete_node(RTBLHost, RTBLDomainsNode, Owner), + disconnect(Config). + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +single_test(T) -> + list_to_atom("antispam_" ++ atom_to_list(T)). + +has_spam_domain(Domain) -> + fun(Host) -> + lists:member(Domain, mod_antispam:get_blocked_domains(Host)) + end. + +is_not_spam(Msg) -> + ?match({Msg, undefined}, mod_antispam:s2s_receive_packet({Msg, undefined})). + +is_spam(Spam) -> + ?match({stop, {drop, undefined}}, mod_antispam:s2s_receive_packet({Spam, undefined})). diff --git a/test/ejabberd_SUITE.erl b/test/ejabberd_SUITE.erl index b0e47cde6..4b2a7fccc 100644 --- a/test/ejabberd_SUITE.erl +++ b/test/ejabberd_SUITE.erl @@ -339,6 +339,10 @@ init_per_testcase(TestCase, OrigConfig) -> bind(auth(connect(Config))); "replaced" ++ _ -> auth(connect(Config)); + "antispam" ++ _ -> + Password = ?config(password, Config), + ejabberd_auth:try_register(User, Server, Password), + open_session(bind(auth(connect(Config)))); _ when IsMaster or IsSlave -> Password = ?config(password, Config), ejabberd_auth:try_register(User, Server, Password), @@ -425,6 +429,7 @@ db_tests(DB) when DB == mnesia; DB == redis -> auth_md5, presence_broadcast, last, + antispam_tests:single_cases(), webadmin_tests:single_cases(), roster_tests:single_cases(), private_tests:single_cases(), diff --git a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml index 42ab23ab2..c8585e6e4 100644 --- a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml +++ b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml @@ -6,6 +6,11 @@ define_macro: mod_announce: db_type: internal access: local + mod_antispam: + rtbl_host: pubsub.mnesia.localhost + spam_jids_file: spam_jids.txt + spam_domains_file: spam_domains.txt + spam_urls_file: spam_urls.txt mod_blocking: [] mod_caps: db_type: internal diff --git a/test/ejabberd_SUITE_data/ejabberd.redis.yml b/test/ejabberd_SUITE_data/ejabberd.redis.yml index 6ff7d7cdb..cdbc905bd 100644 --- a/test/ejabberd_SUITE_data/ejabberd.redis.yml +++ b/test/ejabberd_SUITE_data/ejabberd.redis.yml @@ -7,6 +7,11 @@ define_macro: mod_announce: db_type: internal access: local + mod_antispam: + rtbl_host: pubsub.redis.localhost + spam_jids_file: spam_jids.txt + spam_domains_file: spam_domains.txt + spam_urls_file: spam_urls.txt mod_blocking: [] mod_caps: db_type: internal diff --git a/test/ejabberd_SUITE_data/spam_domains.txt b/test/ejabberd_SUITE_data/spam_domains.txt new file mode 100644 index 000000000..c081998b7 --- /dev/null +++ b/test/ejabberd_SUITE_data/spam_domains.txt @@ -0,0 +1 @@ +spam_domain.org diff --git a/test/ejabberd_SUITE_data/spam_jids.txt b/test/ejabberd_SUITE_data/spam_jids.txt new file mode 100644 index 000000000..2b911fd82 --- /dev/null +++ b/test/ejabberd_SUITE_data/spam_jids.txt @@ -0,0 +1 @@ +spammer_jid@localhost diff --git a/test/ejabberd_SUITE_data/spam_urls.txt b/test/ejabberd_SUITE_data/spam_urls.txt new file mode 100644 index 000000000..857172d46 --- /dev/null +++ b/test/ejabberd_SUITE_data/spam_urls.txt @@ -0,0 +1 @@ +https://spam.domain.url diff --git a/test/suite.erl b/test/suite.erl index 28825b1d1..62b442580 100644 --- a/test/suite.erl +++ b/test/suite.erl @@ -51,6 +51,9 @@ init_config(Config) -> {ok, _} = file:copy(SelfSignedCertFile, filename:join([CWD, "self-signed-cert.pem"])), {ok, _} = file:copy(CAFile, filename:join([CWD, "ca.pem"])), + copy_file(Config, "spam_jids.txt"), + copy_file(Config, "spam_urls.txt"), + copy_file(Config, "spam_domains.txt"), {ok, MacrosContentTpl} = file:read_file(MacrosPathTpl), Password = <<"password!@#$%^&*()'\"`~<>+-/;:_=[]{}|\\">>, Backends = get_config_backends(), @@ -138,6 +141,11 @@ init_config(Config) -> {backends, Backends} |Config]. +copy_file(Config, File) -> + {ok, CWD} = file:get_cwd(), + DataDir = proplists:get_value(data_dir, Config), + {ok, _} = file:copy(filename:join([DataDir, File]), filename:join([CWD, File])). + copy_configtest_yml(DataDir, CWD) -> Files = filelib:wildcard(filename:join([DataDir, "configtest.yml"])), lists:foreach( @@ -906,6 +914,21 @@ receiver(NS, Owner, Socket, MRef) -> receiver(NS, Owner, Socket, MRef) end. +%% @doc Retry an action until success, at max N times with an interval +%% `Interval' +%% Shamlessly stolen (with slight adaptations) from snabbkaffee. +-spec retry(integer(), non_neg_integer(), fun(() -> Ret)) -> Ret. +retry(_, 0, Fun) -> + Fun(); +retry(Interval, N, Fun) -> + try Fun() + catch + EC:Err -> + timer:sleep(Interval), + ct:pal("retrying ~p more times, result was ~p:~p", [N, EC, Err]), + retry(Interval, N - 1, Fun) + end. + %%%=================================================================== %%% Clients puts and gets events via this relay. %%%=================================================================== diff --git a/test/suite.hrl b/test/suite.hrl index 08bb87df1..00b341cb1 100644 --- a/test/suite.hrl +++ b/test/suite.hrl @@ -89,6 +89,8 @@ -define(send_recv(Send, Recv), ?match(Recv, suite:send_recv(Config, Send))). +-define(retry(TIMEOUT, N, FUN), suite:retry(TIMEOUT, N, fun() -> FUN end)). + -define(COMMON_VHOST, <<"localhost">>). -define(MNESIA_VHOST, <<"mnesia.localhost">>). -define(REDIS_VHOST, <<"redis.localhost">>). From 639147be416ed5cf85043551155b4fa19ddec144 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 23 May 2025 17:16:37 +0200 Subject: [PATCH 04/27] fix pubsub retract items being a list of ids --- src/mod_antispam_rtbl.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mod_antispam_rtbl.erl b/src/mod_antispam_rtbl.erl index f9994fd08..d977aa0c6 100644 --- a/src/mod_antispam_rtbl.erl +++ b/src/mod_antispam_rtbl.erl @@ -83,12 +83,8 @@ parse_blocked_domains(#iq{to = #jid{lserver = LServer}, type = result} = IQ) -> parse_pubsub_event(#message{to = #jid{lserver = LServer}} = Msg) -> RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), case xmpp:get_subtag(Msg, #ps_event{}) of - #ps_event{items = #ps_items{node = RTBLDomainsNode, retract = ID}} when ID /= undefined -> - ?DEBUG("Got item to retract:~n~p", [ID]), - #{ID => false}; - #ps_event{items = #ps_items{node = RTBLDomainsNode, items = Items}} -> - ?DEBUG("Got items:~n~p", [Items]), - parse_items(Items); + #ps_event{items = #ps_items{node = RTBLDomainsNode, items = Items, retract = RetractIds}} -> + maps:merge(retract_items(RetractIds), parse_items(Items)); Other -> ?WARNING_MSG("Couldn't extract items: ~p", [Other]), #{} @@ -102,6 +98,10 @@ parse_items(Items) -> maps:put(ID, true, Acc) end, #{}, Items). +-spec retract_items([binary()]) -> #{binary() => false}. +retract_items(Ids) -> + lists:foldl(fun(ID, Acc) -> Acc#{ID => false} end, #{}, Ids). + -spec service_jid(binary()) -> jid(). service_jid(Host) -> jid:make(<<>>, Host, <>). From 34b40aec663b4d08b5a80da7dd8fc84c4e691fdc Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 5 Jun 2025 14:31:02 +0200 Subject: [PATCH 05/27] mod_antispam: add format instructions --- src/mod_antispam.erl | 1007 ++++++++++++++++++++----------------- src/mod_antispam_rtbl.erl | 85 ++-- test/antispam_tests.erl | 102 ++-- 3 files changed, 675 insertions(+), 519 deletions(-) diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index 986f21066..c3489aefa 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -98,6 +98,8 @@ -define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). -define(DEFAULT_CACHE_SIZE, 10000). +%% @format-begin + %%-------------------------------------------------------------------- %% gen_mod callbacks. %%-------------------------------------------------------------------- @@ -114,10 +116,10 @@ start(Host, Opts) -> -spec stop(binary()) -> ok | {error, any()}. stop(Host) -> case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of - false -> - ejabberd_commands:unregister_commands(get_commands_spec()); - true -> - ok + false -> + ejabberd_commands:unregister_commands(get_commands_spec()); + true -> + ok end, gen_mod:stop_child(?MODULE, Host). @@ -134,34 +136,28 @@ depends(_Host, _Opts) -> -spec mod_opt_type(atom()) -> econf:validator(). mod_opt_type(spam_domains_file) -> econf:either( - econf:enum([none]), - econf:file()); + econf:enum([none]), econf:file()); mod_opt_type(whitelist_domains_file) -> - econf:either( - none, - econf:binary()); + econf:either(none, econf:binary()); mod_opt_type(spam_dump_file) -> econf:either( - econf:enum([none]), - econf:binary()); + econf:enum([none]), econf:binary()); mod_opt_type(spam_jids_file) -> econf:either( - econf:enum([none]), - econf:file()); + econf:enum([none]), econf:file()); mod_opt_type(spam_urls_file) -> econf:either( - econf:enum([none]), - econf:file()); + econf:enum([none]), econf:file()); mod_opt_type(access_spam) -> econf:acl(); mod_opt_type(cache_size) -> econf:pos_int(unlimited); mod_opt_type(rtbl_host) -> econf:either( - econf:enum([none]), - econf:host()); + econf:enum([none]), econf:host()); mod_opt_type(rtbl_domains_node) -> - econf:non_empty(econf:binary()). + econf:non_empty( + econf:binary()). -spec mod_options(binary()) -> [{atom(), any()}]. mod_options(_Host) -> @@ -175,7 +171,8 @@ mod_options(_Host) -> {rtbl_host, none}, {rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}]. -mod_doc() -> #{}. +mod_doc() -> + #{}. %%-------------------------------------------------------------------- %% gen_server callbacks. @@ -184,67 +181,73 @@ mod_doc() -> #{}. init([Host, Opts]) -> process_flag(trap_exit, true), DumpFile = expand_host(gen_mod:get_opt(spam_dump_file, Opts), Host), - Files = #{domains => gen_mod:get_opt(spam_domains_file, Opts), - jid => gen_mod:get_opt(spam_jids_file, Opts), - url => gen_mod:get_opt(spam_urls_file, Opts), - whitelist_domains => gen_mod:get_opt(whitelist_domains_file, Opts)}, + Files = + #{domains => gen_mod:get_opt(spam_domains_file, Opts), + jid => gen_mod:get_opt(spam_jids_file, Opts), + url => gen_mod:get_opt(spam_urls_file, Opts), + whitelist_domains => gen_mod:get_opt(whitelist_domains_file, Opts)}, try read_files(Files) of - #{jid := JIDsSet, url := URLsSet, domains := SpamDomainsSet, whitelist_domains := WhitelistDomains} -> - ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, - s2s_in_handle_info, 90), - ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, - s2s_receive_packet, 50), - ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, - sm_receive_packet, 50), - ejabberd_hooks:add(reopen_log_hook, ?MODULE, - reopen_log, 50), - ejabberd_hooks:add(local_send_to_resource_hook, Host, - mod_antispam_rtbl, pubsub_event_handler, 50), - RTBLHost = gen_mod:get_opt(rtbl_host, Opts), - RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), - InitState0 = #state{host = Host, - jid_set = JIDsSet, - url_set = URLsSet, - max_cache_size = gen_mod:get_opt(cache_size, Opts), - blocked_domains = set_to_map(SpamDomainsSet), - whitelist_domains = set_to_map(WhitelistDomains, false), - rtbl_host = RTBLHost, - rtbl_domains_node = RTBLDomainsNode}, - mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), - InitState = init_open_dump_file(DumpFile, InitState0), - {ok, InitState} - catch {Op, File, Reason} when Op == open; - Op == read -> - ?CRITICAL_MSG("Cannot ~s ~s: ~s", [Op, File, format_error(Reason)]), - {stop, config_error} + #{jid := JIDsSet, + url := URLsSet, + domains := SpamDomainsSet, + whitelist_domains := WhitelistDomains} -> + ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90), + ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), + ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50), + ejabberd_hooks:add(reopen_log_hook, ?MODULE, reopen_log, 50), + ejabberd_hooks:add(local_send_to_resource_hook, + Host, + mod_antispam_rtbl, + pubsub_event_handler, + 50), + RTBLHost = gen_mod:get_opt(rtbl_host, Opts), + RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), + InitState0 = + #state{host = Host, + jid_set = JIDsSet, + url_set = URLsSet, + max_cache_size = gen_mod:get_opt(cache_size, Opts), + blocked_domains = set_to_map(SpamDomainsSet), + whitelist_domains = set_to_map(WhitelistDomains, false), + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode}, + mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), + InitState = init_open_dump_file(DumpFile, InitState0), + {ok, InitState} + catch + {Op, File, Reason} when Op == open; Op == read -> + ?CRITICAL_MSG("Cannot ~s ~s: ~s", [Op, File, format_error(Reason)]), + {stop, config_error} end. init_open_dump_file(none, State) -> State; init_open_dump_file(DumpFile, State) -> case filelib:ensure_dir(DumpFile) of - ok -> - ok; - {error, Reason} -> - Dirname = filename:dirname(DumpFile), - throw({open, Dirname, Reason}) + ok -> + ok; + {error, Reason} -> + Dirname = filename:dirname(DumpFile), + throw({open, Dirname, Reason}) end, open_dump_file(DumpFile, State). --spec handle_call(term(), {pid(), term()}, state()) - -> {reply, {spam_filter, term()}, state()} | {noreply, state()}. +-spec handle_call(term(), {pid(), term()}, state()) -> + {reply, {spam_filter, term()}, state()} | {noreply, state()}. handle_call({check_jid, From}, _From, #state{jid_set = JIDsSet} = State) -> {Result, State1} = filter_jid(From, JIDsSet, State), {reply, {spam_filter, Result}, State1}; -handle_call({check_body, URLs, JIDs, From}, _From, - #state{url_set = URLsSet, jid_set = JIDsSet} = State) -> +handle_call({check_body, URLs, JIDs, From}, + _From, + #state{url_set = URLsSet, jid_set = JIDsSet} = State) -> {Result1, State1} = filter_body(URLs, URLsSet, From, State), {Result2, State2} = filter_body(JIDs, JIDsSet, From, State1), - Result = if Result1 == spam -> - Result1; - true -> - Result2 - end, + Result = + if Result1 == spam -> + Result1; + true -> + Result2 + end, {reply, {spam_filter, Result}, State2}; handle_call({resolve_redirects, URLs}, _From, State) -> ResolvedURLs = do_resolve_redirects(URLs, []), @@ -263,18 +266,30 @@ handle_call({drop_from_cache, JID}, _From, State) -> {reply, {spam_filter, Result}, State1}; handle_call(get_cache, _From, #state{jid_cache = Cache} = State) -> {reply, {spam_filter, maps:to_list(Cache)}, State}; -handle_call({add_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains} = State) -> +handle_call({add_blocked_domain, Domain}, + _From, + #state{blocked_domains = BlockedDomains} = State) -> BlockedDomains1 = maps:merge(BlockedDomains, #{Domain => true}), Txt = format("~s added to blocked domains", [Domain]), {reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}}; -handle_call({remove_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains} = State) -> +handle_call({remove_blocked_domain, Domain}, + _From, + #state{blocked_domains = BlockedDomains} = State) -> BlockedDomains1 = maps:remove(Domain, BlockedDomains), Txt = format("~s removed from blocked domains", [Domain]), {reply, {spam_filter, {ok, Txt}}, State#state{blocked_domains = BlockedDomains1}}; -handle_call(get_blocked_domains, _From, #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = State) -> +handle_call(get_blocked_domains, + _From, + #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = + State) -> {reply, {blocked_domains, maps:merge(BlockedDomains, WhitelistDomains)}, State}; -handle_call({is_blocked_domain, Domain}, _From, #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = State) -> - {reply, maps:get(Domain, maps:merge(BlockedDomains, WhitelistDomains), false) =/= false, State}; +handle_call({is_blocked_domain, Domain}, + _From, + #state{blocked_domains = BlockedDomains, whitelist_domains = WhitelistDomains} = + State) -> + {reply, + maps:get(Domain, maps:merge(BlockedDomains, WhitelistDomains), false) =/= false, + State}; handle_call(Request, From, State) -> ?ERROR_MSG("Got unexpected request from ~p: ~p", [From, Request]), {noreply, State}. @@ -284,41 +299,43 @@ handle_cast({dump, _XML}, #state{dump_fd = undefined} = State) -> {noreply, State}; handle_cast({dump, XML}, #state{dump_fd = Fd} = State) -> case file:write(Fd, [XML, <<$\n>>]) of - ok -> - ok; - {error, Reason} -> - ?ERROR_MSG("Cannot write spam to dump file: ~s", - [file:format_error(Reason)]) + ok -> + ok; + {error, Reason} -> + ?ERROR_MSG("Cannot write spam to dump file: ~s", [file:format_error(Reason)]) end, {noreply, State}; handle_cast({reload, NewOpts, OldOpts}, - #state{host = Host, - rtbl_host = OldRTBLHost, - rtbl_domains_node = OldRTBLDomainsNode, - rtbl_retry_timer = RTBLRetryTimer} = State) -> + #state{host = Host, + rtbl_host = OldRTBLHost, + rtbl_domains_node = OldRTBLDomainsNode, + rtbl_retry_timer = RTBLRetryTimer} = + State) -> misc:cancel_timer(RTBLRetryTimer), - State1 = case {gen_mod:get_opt(spam_dump_file, OldOpts), - gen_mod:get_opt(spam_dump_file, NewOpts)} of - {OldDumpFile, NewDumpFile} when NewDumpFile /= OldDumpFile -> - close_dump_file(expand_host(OldDumpFile, Host), State), - open_dump_file(expand_host(NewDumpFile, Host), State); - {_OldDumpFile, _NewDumpFile} -> - State - end, - State2 = case {gen_mod:get_opt(cache_size, OldOpts), - gen_mod:get_opt(cache_size, NewOpts)} of - {OldMax, NewMax} when NewMax < OldMax -> - shrink_cache(State1#state{max_cache_size = NewMax}); - {OldMax, NewMax} when NewMax > OldMax -> - State1#state{max_cache_size = NewMax}; - {_OldMax, _NewMax} -> - State1 - end, + State1 = + case {gen_mod:get_opt(spam_dump_file, OldOpts), gen_mod:get_opt(spam_dump_file, NewOpts)} + of + {OldDumpFile, NewDumpFile} when NewDumpFile /= OldDumpFile -> + close_dump_file(expand_host(OldDumpFile, Host), State), + open_dump_file(expand_host(NewDumpFile, Host), State); + {_OldDumpFile, _NewDumpFile} -> + State + end, + State2 = + case {gen_mod:get_opt(cache_size, OldOpts), gen_mod:get_opt(cache_size, NewOpts)} of + {OldMax, NewMax} when NewMax < OldMax -> + shrink_cache(State1#state{max_cache_size = NewMax}); + {OldMax, NewMax} when NewMax > OldMax -> + State1#state{max_cache_size = NewMax}; + {_OldMax, _NewMax} -> + State1 + end, ok = mod_antispam_rtbl:unsubscribe(OldRTBLHost, OldRTBLDomainsNode, Host), - Files = #{domains => gen_mod:get_opt(spam_domains_file, NewOpts), - jid => gen_mod:get_opt(spam_jids_file, NewOpts), - url => gen_mod:get_opt(spam_urls_file, NewOpts), - whitelist_domains => gen_mod:get_opt(whitelist_domains_file, NewOpts)}, + Files = + #{domains => gen_mod:get_opt(spam_domains_file, NewOpts), + jid => gen_mod:get_opt(spam_jids_file, NewOpts), + url => gen_mod:get_opt(spam_urls_file, NewOpts), + whitelist_domains => gen_mod:get_opt(whitelist_domains_file, NewOpts)}, {_Result, State3} = reload_files(Files, State2#state{blocked_domains = #{}}), RTBLHost = gen_mod:get_opt(rtbl_host, NewOpts), RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, NewOpts), @@ -326,7 +343,8 @@ handle_cast({reload, NewOpts, OldOpts}, {noreply, State3#state{rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode}}; handle_cast(reopen_log, State) -> {noreply, reopen_dump_file(State)}; -handle_cast({update_blocked_domains, NewItems}, #state{blocked_domains = BlockedDomains} = State) -> +handle_cast({update_blocked_domains, NewItems}, + #state{blocked_domains = BlockedDomains} = State) -> {noreply, State#state{blocked_domains = maps:merge(BlockedDomains, NewItems)}}; handle_cast(Request, State) -> ?ERROR_MSG("Got unexpected request from: ~p", [Request]), @@ -334,32 +352,42 @@ handle_cast(Request, State) -> -spec handle_info(term(), state()) -> {noreply, state()}. handle_info({iq_reply, timeout, blocked_domains}, State) -> - ?WARNING_MSG("Fetching blocked domains failed: fetch timeout. Retrying in 60 seconds", []), - {noreply, State#state{rtbl_retry_timer = erlang:send_after(60000, self(), request_blocked_domains)}}; + ?WARNING_MSG("Fetching blocked domains failed: fetch timeout. Retrying in 60 seconds", + []), + {noreply, + State#state{rtbl_retry_timer = + erlang:send_after(60000, self(), request_blocked_domains)}}; handle_info({iq_reply, #iq{type = error} = IQ, blocked_domains}, State) -> ?WARNING_MSG("Fetching blocked domains failed: ~p. Retrying in 60 seconds", - [xmpp:format_stanza_error(xmpp:get_error(IQ))]), - {noreply, State#state{rtbl_retry_timer = erlang:send_after(60000, self(), request_blocked_domains)}}; + [xmpp:format_stanza_error( + xmpp:get_error(IQ))]), + {noreply, + State#state{rtbl_retry_timer = + erlang:send_after(60000, self(), request_blocked_domains)}}; handle_info({iq_reply, IQReply, blocked_domains}, - #state{blocked_domains = OldBlockedDomains, - rtbl_host = RTBLHost, - rtbl_domains_node = RTBLDomainsNode, - host = Host} = State) -> + #state{blocked_domains = OldBlockedDomains, + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode, + host = Host} = + State) -> case mod_antispam_rtbl:parse_blocked_domains(IQReply) of - undefined -> - ?WARNING_MSG("Fetching initial list failed: invalid result payload", []), - {noreply, State#state{rtbl_retry_timer = undefined}}; - NewBlockedDomains -> - ok = mod_antispam_rtbl:subscribe(RTBLHost, RTBLDomainsNode, Host), - {noreply, State#state{rtbl_retry_timer = undefined, - rtbl_subscribed = true, - blocked_domains = maps:merge(OldBlockedDomains, NewBlockedDomains)}} + undefined -> + ?WARNING_MSG("Fetching initial list failed: invalid result payload", []), + {noreply, State#state{rtbl_retry_timer = undefined}}; + NewBlockedDomains -> + ok = mod_antispam_rtbl:subscribe(RTBLHost, RTBLDomainsNode, Host), + {noreply, + State#state{rtbl_retry_timer = undefined, + rtbl_subscribed = true, + blocked_domains = maps:merge(OldBlockedDomains, NewBlockedDomains)}} end; handle_info({iq_reply, timeout, subscribe_result}, State) -> ?WARNING_MSG("Subscription error: request timeout", []), {noreply, State#state{rtbl_subscribed = false}}; handle_info({iq_reply, #iq{type = error} = IQ, subscribe_result}, State) -> - ?WARNING_MSG("Subscription error: ~p", [xmpp:format_stanza_error(xmpp:get_error(IQ))]), + ?WARNING_MSG("Subscription error: ~p", + [xmpp:format_stanza_error( + xmpp:get_error(IQ))]), {noreply, State#state{rtbl_subscribed = false}}; handle_info({iq_reply, IQReply, subscribe_result}, State) -> ?DEBUG("Got subscribe result: ~p", [IQReply]), @@ -368,7 +396,11 @@ handle_info({iq_reply, _IQReply, unsubscribe_result}, State) -> %% FIXME: we should check it's true (of type `result`, not `error`), but at that point, what %% would we do? {noreply, State#state{rtbl_subscribed = false}}; -handle_info(request_blocked_domains, #state{host = Host, rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode} = State) -> +handle_info(request_blocked_domains, + #state{host = Host, + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode} = + State) -> mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), {noreply, State}; handle_info(Info, State) -> @@ -377,30 +409,30 @@ handle_info(Info, State) -> -spec terminate(normal | shutdown | {shutdown, term()} | term(), state()) -> ok. terminate(Reason, - #state{host = Host, - rtbl_host = RTBLHost, - rtbl_domains_node = RTBLDomainsNode, - rtbl_retry_timer = RTBLRetryTimer} = State) -> + #state{host = Host, + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode, + rtbl_retry_timer = RTBLRetryTimer} = + State) -> ?DEBUG("Stopping spam filter process for ~s: ~p", [Host, Reason]), misc:cancel_timer(RTBLRetryTimer), DumpFile = gen_mod:get_module_opt(Host, ?MODULE, spam_dump_file), DumpFile1 = expand_host(DumpFile, Host), close_dump_file(DumpFile1, State), - ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, - s2s_receive_packet, 50), - ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, - sm_receive_packet, 50), - ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, - s2s_in_handle_info, 90), - ejabberd_hooks:delete(local_send_to_resource_hook, Host, - mod_antispam_rtbl, pubsub_event_handler, 50), - mod_antispam_rtbl:unsubscribe(RTBLHost, RTBLDomainsNode,Host), + ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), + ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50), + ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90), + ejabberd_hooks:delete(local_send_to_resource_hook, + Host, + mod_antispam_rtbl, + pubsub_event_handler, + 50), + mod_antispam_rtbl:unsubscribe(RTBLHost, RTBLDomainsNode, Host), case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of - false -> - ejabberd_hooks:delete(reopen_log_hook, ?MODULE, - reopen_log, 50); - true -> - ok + false -> + ejabberd_hooks:delete(reopen_log_hook, ?MODULE, reopen_log, 50); + true -> + ok end. -spec code_change({down, term()} | term(), state(), term()) -> {ok, state()}. @@ -412,8 +444,8 @@ code_change(_OldVsn, #state{host = Host} = State, _Extra) -> %% Hook callbacks. %%-------------------------------------------------------------------- --spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) - -> {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}. +-spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) -> + {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}. s2s_receive_packet({A, State}) -> case sm_receive_packet(A) of {stop, drop} -> @@ -426,36 +458,37 @@ s2s_receive_packet({A, State}) -> sm_receive_packet(drop = Acc) -> Acc; sm_receive_packet(#message{from = From, - to = #jid{lserver = LServer} = To, - type = Type} = Msg) - when Type /= groupchat, - Type /= error -> + to = #jid{lserver = LServer} = To, + type = Type} = + Msg) + when Type /= groupchat, Type /= error -> do_check(From, To, LServer, Msg); sm_receive_packet(#presence{from = From, - to = #jid{lserver = LServer} = To, - type = subscribe} = Presence) -> + to = #jid{lserver = LServer} = To, + type = subscribe} = + Presence) -> do_check(From, To, LServer, Presence); sm_receive_packet(Acc) -> Acc. do_check(From, To, LServer, Stanza) -> case needs_checking(From, To) of - true -> - case check_from(LServer, From) of - ham -> - case check_stanza(LServer, From, Stanza) of - ham -> - Stanza; - spam -> - reject(Stanza), - {stop, drop} - end; - spam -> - reject(Stanza), - {stop, drop} - end; - false -> - Stanza + true -> + case check_from(LServer, From) of + ham -> + case check_stanza(LServer, From, Stanza) of + ham -> + Stanza; + spam -> + reject(Stanza), + {stop, drop} + end; + spam -> + reject(Stanza), + {stop, drop} + end; + false -> + Stanza end. check_stanza(LServer, From, #message{body = Body}) -> @@ -463,8 +496,8 @@ check_stanza(LServer, From, #message{body = Body}) -> check_stanza(_, _, _) -> ham. --spec s2s_in_handle_info(s2s_in_state(), any()) - -> s2s_in_state() | {stop, s2s_in_state()}. +-spec s2s_in_handle_info(s2s_in_state(), any()) -> + s2s_in_state() | {stop, s2s_in_state()}. s2s_in_handle_info(State, {_Ref, {spam_filter, _}}) -> ?DEBUG("Dropping expired spam filter result", []), {stop, State}; @@ -474,9 +507,10 @@ s2s_in_handle_info(State, _) -> -spec reopen_log() -> ok. reopen_log() -> lists:foreach(fun(Host) -> - Proc = get_proc_name(Host), - gen_server:cast(Proc, reopen_log) - end, get_spam_filter_hosts()). + Proc = get_proc_name(Host), + gen_server:cast(Proc, reopen_log) + end, + get_spam_filter_hosts()). %%-------------------------------------------------------------------- %% Internal functions. @@ -484,60 +518,70 @@ reopen_log() -> -spec needs_checking(jid(), jid()) -> boolean(). needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) -> case gen_mod:is_loaded(LServer, ?MODULE) of - true -> - Access = gen_mod:get_module_opt(LServer, ?MODULE, access_spam), - case acl:match_rule(LServer, Access, To) of - allow -> - ?DEBUG("Spam not filtered for ~s", [jid:encode(To)]), - false; - deny -> - ?DEBUG("Spam is filtered for ~s", [jid:encode(To)]), - not mod_roster:is_subscribed(From, To) andalso - not mod_roster:is_subscribed(jid:make(<<>>, FromHost), To) % likely a gateway - end; - false -> - ?DEBUG("~s not loaded for ~s", [?MODULE, LServer]), - false + true -> + Access = gen_mod:get_module_opt(LServer, ?MODULE, access_spam), + case acl:match_rule(LServer, Access, To) of + allow -> + ?DEBUG("Spam not filtered for ~s", [jid:encode(To)]), + false; + deny -> + ?DEBUG("Spam is filtered for ~s", [jid:encode(To)]), + not mod_roster:is_subscribed(From, To) + andalso not + mod_roster:is_subscribed( + jid:make(<<>>, FromHost), + To) % likely a gateway + end; + false -> + ?DEBUG("~s not loaded for ~s", [?MODULE, LServer]), + false end. -spec check_from(binary(), jid()) -> ham | spam. check_from(Host, From) -> Proc = get_proc_name(Host), - LFrom = {_, FromDomain, _} = jid:remove_resource(jid:tolower(From)), + LFrom = + {_, FromDomain, _} = + jid:remove_resource( + jid:tolower(From)), try - case gen_server:call(Proc, {is_blocked_domain, FromDomain}) of - true -> + case gen_server:call(Proc, {is_blocked_domain, FromDomain}) of + true -> ?DEBUG("Spam JID found in blocked domains: ~p", [From]), ejabberd_hooks:run(spam_found, Host, [{jid, From}]), spam; - false -> - case gen_server:call(Proc, {check_jid, LFrom}) of - {spam_filter, Result} -> - Result - end - end - catch exit:{timeout, _} -> - ?WARNING_MSG("Timeout while checking ~s against list of blocked domains or spammers", - [jid:encode(From)]), - ham + false -> + case gen_server:call(Proc, {check_jid, LFrom}) of + {spam_filter, Result} -> + Result + end + end + catch + exit:{timeout, _} -> + ?WARNING_MSG("Timeout while checking ~s against list of blocked domains or spammers", + [jid:encode(From)]), + ham end. -spec check_body(binary(), jid(), binary()) -> ham | spam. check_body(Host, From, Body) -> case {extract_urls(Host, Body), extract_jids(Body)} of - {none, none} -> - ?DEBUG("No JIDs/URLs found in message", []), - ham; - {URLs, JIDs} -> - Proc = get_proc_name(Host), - LFrom = jid:remove_resource(jid:tolower(From)), - try gen_server:call(Proc, {check_body, URLs, JIDs, LFrom}) of - {spam_filter, Result} -> - Result - catch exit:{timeout, _} -> - ?WARNING_MSG("Timeout while checking body", []), - ham - end + {none, none} -> + ?DEBUG("No JIDs/URLs found in message", []), + ham; + {URLs, JIDs} -> + Proc = get_proc_name(Host), + LFrom = + jid:remove_resource( + jid:tolower(From)), + try gen_server:call(Proc, {check_body, URLs, JIDs, LFrom}) of + {spam_filter, Result} -> + Result + catch + exit:{timeout, _} -> + ?WARNING_MSG("Timeout while checking body", []), + ham + end end. -spec extract_urls(binary(), binary()) -> {urls, [url()]} | none. @@ -545,11 +589,11 @@ extract_urls(Host, Body) -> RE = <<"https?://\\S+">>, Options = [global, {capture, all, binary}], case re:run(Body, RE, Options) of - {match, Captured} when is_list(Captured) -> - Urls = resolve_redirects(Host, lists:flatten(Captured)), - {urls, Urls}; - nomatch -> - none + {match, Captured} when is_list(Captured) -> + Urls = resolve_redirects(Host, lists:flatten(Captured)), + {urls, Urls}; + nomatch -> + none end. -spec resolve_redirects(binary(), [url()]) -> [url()]. @@ -558,17 +602,20 @@ resolve_redirects(Host, URLs) -> try gen_server:call(Proc, {resolve_redirects, URLs}) of {spam_filter, ResolvedURLs} -> ResolvedURLs - catch exit:{timeout, _} -> + catch + exit:{timeout, _} -> ?WARNING_MSG("Timeout while resolving redirects: ~p", [URLs]), URLs end. -spec do_resolve_redirects([url()], [url()]) -> [url()]. -do_resolve_redirects([], Result) -> Result; +do_resolve_redirects([], Result) -> + Result; do_resolve_redirects([URL | Rest], Acc) -> - case - httpc:request(get, {URL, [{"user-agent", "curl/8.7.1"}]}, - [{autoredirect, false}, {timeout, ?HTTPC_TIMEOUT}], []) + case httpc:request(get, + {URL, [{"user-agent", "curl/8.7.1"}]}, + [{autoredirect, false}, {timeout, ?HTTPC_TIMEOUT}], + []) of {ok, {{_, StatusCode, _}, Headers, _Body}} when StatusCode >= 300, StatusCode < 400 -> Location = proplists:get_value("location", Headers), @@ -577,7 +624,7 @@ do_resolve_redirects([URL | Rest], Acc) -> do_resolve_redirects(Rest, [URL | Acc]); false -> do_resolve_redirects([Location | Rest], [URL | Acc]) - end; + end; _Res -> do_resolve_redirects(Rest, [URL | Acc]) end. @@ -587,85 +634,91 @@ extract_jids(Body) -> RE = <<"\\S+@\\S+">>, Options = [global, {capture, all, binary}], case re:run(Body, RE, Options) of - {match, Captured} when is_list(Captured) -> - {jids, lists:filtermap(fun try_decode_jid/1, - lists:flatten(Captured))}; - nomatch -> - none + {match, Captured} when is_list(Captured) -> + {jids, lists:filtermap(fun try_decode_jid/1, lists:flatten(Captured))}; + nomatch -> + none end. -spec try_decode_jid(binary()) -> {true, ljid()} | false. try_decode_jid(S) -> try jid:decode(S) of - #jid{} = JID -> - {true, jid:remove_resource(jid:tolower(JID))} - catch _:{bad_jid, _} -> - false + #jid{} = JID -> + {true, + jid:remove_resource( + jid:tolower(JID))} + catch + _:{bad_jid, _} -> + false end. -spec filter_jid(ljid(), jid_set(), state()) -> {ham | spam, state()}. filter_jid(From, Set, #state{host = Host} = State) -> case sets:is_element(From, Set) of - true -> - ?DEBUG("Spam JID found: ~s", [jid:encode(From)]), + true -> + ?DEBUG("Spam JID found: ~s", [jid:encode(From)]), ejabberd_hooks:run(spam_found, Host, [{jid, From}]), - {spam, State}; - false -> - case cache_lookup(From, State) of - {true, State1} -> - ?DEBUG("Spam JID found: ~s", [jid:encode(From)]), + {spam, State}; + false -> + case cache_lookup(From, State) of + {true, State1} -> + ?DEBUG("Spam JID found: ~s", [jid:encode(From)]), ejabberd_hooks:run(spam_found, Host, [{jid, From}]), - {spam, State1}; - {false, State1} -> - ?DEBUG("JID not listed: ~s", [jid:encode(From)]), - {ham, State1} - end + {spam, State1}; + {false, State1} -> + ?DEBUG("JID not listed: ~s", [jid:encode(From)]), + {ham, State1} + end end. -spec filter_body({urls, [url()]} | {jids, [ljid()]} | none, - url_set() | jid_set(), jid(), state()) - -> {ham | spam, state()}. + url_set() | jid_set(), + jid(), + state()) -> + {ham | spam, state()}. filter_body({_, Addrs}, Set, From, #state{host = Host} = State) -> case lists:any(fun(Addr) -> sets:is_element(Addr, Set) end, Addrs) of - true -> - ?DEBUG("Spam addresses found: ~p", [Addrs]), + true -> + ?DEBUG("Spam addresses found: ~p", [Addrs]), ejabberd_hooks:run(spam_found, Host, [{body, Addrs}]), - {spam, cache_insert(From, State)}; - false -> - ?DEBUG("Addresses not listed: ~p", [Addrs]), - {ham, State} + {spam, cache_insert(From, State)}; + false -> + ?DEBUG("Addresses not listed: ~p", [Addrs]), + {ham, State} end; filter_body(none, _Set, _From, State) -> {ham, State}. --spec reload_files(#{Type :: atom() => filename()}, state()) - -> {ok | {error, binary()}, state()}. +-spec reload_files(#{Type :: atom() => filename()}, state()) -> + {ok | {error, binary()}, state()}. reload_files(Files, #state{host = Host, blocked_domains = BlockedDomains} = State) -> try read_files(Files) of - #{jid := JIDsSet, url := URLsSet, domains := SpamDomainsSet, whitelist_domains := WhitelistDomains} -> - case sets_equal(JIDsSet, State#state.jid_set) of - true -> - ?INFO_MSG("Reloaded spam JIDs for ~s (unchanged)", [Host]); - false -> - ?INFO_MSG("Reloaded spam JIDs for ~s (changed)", [Host]) - end, - case sets_equal(URLsSet, State#state.url_set) of - true -> - ?INFO_MSG("Reloaded spam URLs for ~s (unchanged)", [Host]); - false -> - ?INFO_MSG("Reloaded spam URLs for ~s (changed)", [Host]) - end, - {ok, State#state{jid_set = JIDsSet, - url_set = URLsSet, - blocked_domains = maps:merge(BlockedDomains, set_to_map(SpamDomainsSet)), - whitelist_domains = set_to_map(WhitelistDomains, false)} - } - catch {Op, File, Reason} when Op == open; - Op == read -> - Txt = format("Cannot ~s ~s for ~s: ~s", - [Op, File, Host, format_error(Reason)]), - ?ERROR_MSG("~s", [Txt]), - {{error, Txt}, State} + #{jid := JIDsSet, + url := URLsSet, + domains := SpamDomainsSet, + whitelist_domains := WhitelistDomains} -> + case sets_equal(JIDsSet, State#state.jid_set) of + true -> + ?INFO_MSG("Reloaded spam JIDs for ~s (unchanged)", [Host]); + false -> + ?INFO_MSG("Reloaded spam JIDs for ~s (changed)", [Host]) + end, + case sets_equal(URLsSet, State#state.url_set) of + true -> + ?INFO_MSG("Reloaded spam URLs for ~s (unchanged)", [Host]); + false -> + ?INFO_MSG("Reloaded spam URLs for ~s (changed)", [Host]) + end, + {ok, + State#state{jid_set = JIDsSet, + url_set = URLsSet, + blocked_domains = maps:merge(BlockedDomains, set_to_map(SpamDomainsSet)), + whitelist_domains = set_to_map(WhitelistDomains, false)}} + catch + {Op, File, Reason} when Op == open; Op == read -> + Txt = format("Cannot ~s ~s for ~s: ~s", [Op, File, Host, format_error(Reason)]), + ?ERROR_MSG("~s", [Txt]), + {{error, Txt}, State} end. set_to_map(Set) -> @@ -674,54 +727,63 @@ set_to_map(Set) -> set_to_map(Set, V) -> sets:fold(fun(K, M) -> M#{K => V} end, #{}, Set). --spec read_files(#{Type => filename()}) -> #{jid => jid_set(), url => url_set(), Type => sets:set(binary())} - when Type :: atom(). +-spec read_files(#{Type => filename()}) -> + #{jid => jid_set(), + url => url_set(), + Type => sets:set(binary())} + when Type :: atom(). read_files(Files) -> - maps:map(fun(Type, Filename) -> - read_file(Filename, line_parser(Type)) - end, - Files). + maps:map(fun(Type, Filename) -> read_file(Filename, line_parser(Type)) end, Files). -spec line_parser(Type :: atom()) -> fun((binary()) -> binary()). -line_parser(jid) -> fun parse_jid/1; -line_parser(url) -> fun parse_url/1; -line_parser(_) -> fun trim/1. +line_parser(jid) -> + fun parse_jid/1; +line_parser(url) -> + fun parse_url/1; +line_parser(_) -> + fun trim/1. --spec read_file(filename(), fun((binary()) -> ljid() | url())) - -> jid_set() | url_set(). +-spec read_file(filename(), fun((binary()) -> ljid() | url())) -> jid_set() | url_set(). read_file(none, _ParseLine) -> sets:new(); read_file(File, ParseLine) -> case file:open(File, [read, binary, raw, {read_ahead, 65536}]) of - {ok, Fd} -> - try read_line(Fd, ParseLine, sets:new()) - catch throw:E -> throw({read, File, E}) - after ok = file:close(Fd) - end; - {error, Reason} -> - throw({open, File, Reason}) + {ok, Fd} -> + try + read_line(Fd, ParseLine, sets:new()) + catch + E -> + throw({read, File, E}) + after + ok = file:close(Fd) + end; + {error, Reason} -> + throw({open, File, Reason}) end. --spec read_line(file:io_device(), fun((binary()) -> ljid() | url()), - jid_set() | url_set()) - -> jid_set() | url_set(). +-spec read_line(file:io_device(), + fun((binary()) -> ljid() | url()), + jid_set() | url_set()) -> + jid_set() | url_set(). read_line(Fd, ParseLine, Set) -> case file:read_line(Fd) of - {ok, Line} -> - read_line(Fd, ParseLine, sets:add_element(ParseLine(Line), Set)); - {error, Reason} -> - throw(Reason); - eof -> - Set + {ok, Line} -> + read_line(Fd, ParseLine, sets:add_element(ParseLine(Line), Set)); + {error, Reason} -> + throw(Reason); + eof -> + Set end. -spec parse_jid(binary()) -> ljid(). parse_jid(S) -> try jid:decode(trim(S)) of - #jid{} = JID -> - jid:remove_resource(jid:tolower(JID)) - catch _:{bad_jid, _} -> - throw({bad_jid, S}) + #jid{} = JID -> + jid:remove_resource( + jid:tolower(JID)) + catch + _:{bad_jid, _} -> + throw({bad_jid, S}) end. -spec parse_url(binary()) -> url(). @@ -730,10 +792,10 @@ parse_url(S) -> RE = <<"https?://\\S+$">>, Options = [anchored, caseless, {capture, none}], case re:run(URL, RE, Options) of - match -> - URL; - nomatch -> - throw({bad_url, S}) + match -> + URL; + nomatch -> + throw({bad_url, S}) end. -spec trim(binary()) -> binary(). @@ -741,18 +803,24 @@ trim(S) -> re:replace(S, <<"\\s+$">>, <<>>, [{return, binary}]). -spec reject(stanza()) -> ok. -reject(#message{from = From, to = To, type = Type, lang = Lang} = Msg) - when Type /= groupchat, - Type /= error -> +reject(#message{from = From, + to = To, + type = Type, + lang = Lang} = + Msg) + when Type /= groupchat, Type /= error -> ?INFO_MSG("Rejecting unsolicited message from ~s to ~s", - [jid:encode(From), jid:encode(To)]), + [jid:encode(From), jid:encode(To)]), Txt = <<"Your message is unsolicited">>, Err = xmpp:err_policy_violation(Txt, Lang), maybe_dump_spam(Msg), ejabberd_router:route_error(Msg, Err); -reject(#presence{from = From, to = To, lang = Lang} = Presence) -> +reject(#presence{from = From, + to = To, + lang = Lang} = + Presence) -> ?INFO_MSG("Rejecting unsolicited presence from ~s to ~s", - [jid:encode(From), jid:encode(To)]), + [jid:encode(From), jid:encode(To)]), Txt = <<"Your traffic is unsolicited">>, Err = xmpp:err_policy_violation(Txt, Lang), ejabberd_router:route_error(Presence, Err); @@ -765,12 +833,12 @@ open_dump_file(none, State) -> open_dump_file(Name, State) -> Modes = [append, raw, binary, delayed_write], case file:open(Name, Modes) of - {ok, Fd} -> - ?DEBUG("Opened ~s", [Name]), - State#state{dump_fd = Fd}; - {error, Reason} -> - ?ERROR_MSG("Cannot open dump file ~s: ~s", [Name, file:format_error(Reason)]), - State#state{dump_fd = undefined} + {ok, Fd} -> + ?DEBUG("Opened ~s", [Name]), + State#state{dump_fd = Fd}; + {error, Reason} -> + ?ERROR_MSG("Cannot open dump file ~s: ~s", [Name, file:format_error(Reason)]), + State#state{dump_fd = undefined} end. -spec close_dump_file(filename(), state()) -> ok. @@ -778,10 +846,10 @@ close_dump_file(_Name, #state{dump_fd = undefined}) -> ok; close_dump_file(Name, #state{dump_fd = Fd}) -> case file:close(Fd) of - ok -> - ?DEBUG("Closed ~s", [Name]); - {error, Reason} -> - ?ERROR_MSG("Cannot close ~s: ~s", [Name, file:format_error(Reason)]) + ok -> + ?DEBUG("Closed ~s", [Name]); + {error, Reason} -> + ?ERROR_MSG("Cannot close ~s: ~s", [Name, file:format_error(Reason)]) end. -spec reopen_dump_file(state()) -> state(). @@ -797,7 +865,8 @@ maybe_dump_spam(#message{to = #jid{lserver = LServer}} = Msg) -> Proc = get_proc_name(LServer), Time = erlang:timestamp(), Msg1 = misc:add_delay_info(Msg, By, Time), - XML = fxml:element_to_binary(xmpp:encode(Msg1)), + XML = fxml:element_to_binary( + xmpp:encode(Msg1)), gen_server:cast(Proc, {dump, XML}). -spec get_proc_name(binary()) -> atom(). @@ -837,7 +906,7 @@ format_error(Reason) -> cache_insert(_LJID, #state{max_cache_size = 0} = State) -> State; cache_insert(LJID, #state{jid_cache = Cache, max_cache_size = MaxSize} = State) - when MaxSize /= unlimited, map_size(Cache) >= MaxSize -> + when MaxSize /= unlimited, map_size(Cache) >= MaxSize -> cache_insert(LJID, shrink_cache(State)); cache_insert(LJID, #state{jid_cache = Cache} = State) -> ?INFO_MSG("Caching spam JID: ~s", [jid:encode(LJID)]), @@ -847,12 +916,12 @@ cache_insert(LJID, #state{jid_cache = Cache} = State) -> -spec cache_lookup(ljid(), state()) -> {boolean(), state()}. cache_lookup(LJID, #state{jid_cache = Cache} = State) -> case Cache of - #{LJID := _Timestamp} -> - Cache1 = Cache#{LJID => erlang:monotonic_time(second)}, - State1 = State#state{jid_cache = Cache1}, - {true, State1}; - #{} -> - {false, State} + #{LJID := _Timestamp} -> + Cache1 = Cache#{LJID => erlang:monotonic_time(second)}, + State1 = State#state{jid_cache = Cache1}, + {true, State1}; + #{} -> + {false, State} end. -spec shrink_cache(state()) -> state(). @@ -860,7 +929,9 @@ shrink_cache(#state{jid_cache = Cache, max_cache_size = MaxSize} = State) -> ShrinkedSize = round(MaxSize / 2), N = map_size(Cache) - ShrinkedSize, L = lists:keysort(2, maps:to_list(Cache)), - Cache1 = maps:from_list(lists:nthtail(N, L)), + Cache1 = + maps:from_list( + lists:nthtail(N, L)), State#state{jid_cache = Cache1}. -spec expire_cache(integer(), state()) -> {{ok, binary()}, state()}. @@ -881,11 +952,11 @@ add_to_cache(LJID, State) -> drop_from_cache(LJID, #state{jid_cache = Cache} = State) -> Cache1 = maps:remove(LJID, Cache), if map_size(Cache1) < map_size(Cache) -> - Txt = format("~s removed from cache", [jid:encode(LJID)]), - {{ok, Txt}, State#state{jid_cache = Cache1}}; + Txt = format("~s removed from cache", [jid:encode(LJID)]), + {{ok, Txt}, State#state{jid_cache = Cache1}}; true -> - Txt = format("~s wasn't cached", [jid:encode(LJID)]), - {{ok, Txt}, State} + Txt = format("~s wasn't cached", [jid:encode(LJID)]), + {{ok, Txt}, State} end. %%-------------------------------------------------------------------- @@ -893,78 +964,96 @@ drop_from_cache(LJID, #state{jid_cache = Cache} = State) -> %%-------------------------------------------------------------------- -spec get_commands_spec() -> [ejabberd_commands()]. get_commands_spec() -> - [#ejabberd_commands{name = reload_spam_filter_files, tags = [filter], - desc = "Reload spam JID/URL files", - module = ?MODULE, function = reload_spam_filter_files, - args = [{host, binary}], - result = {res, rescode}}, - #ejabberd_commands{name = get_spam_filter_cache, tags = [filter], - desc = "Show spam filter cache contents", - module = ?MODULE, function = get_spam_filter_cache, - args = [{host, binary}], - result = {spammers, {list, {spammer, {tuple, - [{jid, string}, {timestamp, integer}]}}}}}, - #ejabberd_commands{name = expire_spam_filter_cache, tags = [filter], - desc = "Remove old/unused spam JIDs from cache", - module = ?MODULE, function = expire_spam_filter_cache, - args = [{host, binary}, {seconds, integer}], - result = {res, restuple}}, - #ejabberd_commands{name = add_to_spam_filter_cache, tags = [filter], - desc = "Add JID to spam filter cache", - module = ?MODULE, - function = add_to_spam_filter_cache, - args = [{host, binary}, {jid, binary}], - result = {res, restuple}}, - #ejabberd_commands{name = drop_from_spam_filter_cache, tags = [filter], - desc = "Drop JID from spam filter cache", - module = ?MODULE, - function = drop_from_spam_filter_cache, - args = [{host, binary}, {jid, binary}], - result = {res, restuple}}, - #ejabberd_commands{name = get_blocked_domains, tags = [filter], - desc = "Get list of domains being blocked", - module = ?MODULE, - function = get_blocked_domains, - args = [{host, binary}], - result = {blocked_domains, {list, {jid, string}}}}, - #ejabberd_commands{name = add_blocked_domain, tags = [filter], - desc = "Add domain to list of blocked domains", - module = ?MODULE, - function = add_blocked_domain, - args = [{host, binary}, {domain, binary}], - result = {res, restuple}}, - #ejabberd_commands{name = remove_blocked_domain, tags = [filter], - desc = "Remove domain from list of blocked domains", - module = ?MODULE, - function = remove_blocked_domain, - args = [{host, binary}, {domain, binary}], - result = {res, restuple}} - ]. + [#ejabberd_commands{name = reload_spam_filter_files, + tags = [filter], + desc = "Reload spam JID/URL files", + module = ?MODULE, + function = reload_spam_filter_files, + args = [{host, binary}], + result = {res, rescode}}, + #ejabberd_commands{name = get_spam_filter_cache, + tags = [filter], + desc = "Show spam filter cache contents", + module = ?MODULE, + function = get_spam_filter_cache, + args = [{host, binary}], + result = + {spammers, + {list, {spammer, {tuple, [{jid, string}, {timestamp, integer}]}}}}}, + #ejabberd_commands{name = expire_spam_filter_cache, + tags = [filter], + desc = "Remove old/unused spam JIDs from cache", + module = ?MODULE, + function = expire_spam_filter_cache, + args = [{host, binary}, {seconds, integer}], + result = {res, restuple}}, + #ejabberd_commands{name = add_to_spam_filter_cache, + tags = [filter], + desc = "Add JID to spam filter cache", + module = ?MODULE, + function = add_to_spam_filter_cache, + args = [{host, binary}, {jid, binary}], + result = {res, restuple}}, + #ejabberd_commands{name = drop_from_spam_filter_cache, + tags = [filter], + desc = "Drop JID from spam filter cache", + module = ?MODULE, + function = drop_from_spam_filter_cache, + args = [{host, binary}, {jid, binary}], + result = {res, restuple}}, + #ejabberd_commands{name = get_blocked_domains, + tags = [filter], + desc = "Get list of domains being blocked", + module = ?MODULE, + function = get_blocked_domains, + args = [{host, binary}], + result = {blocked_domains, {list, {jid, string}}}}, + #ejabberd_commands{name = add_blocked_domain, + tags = [filter], + desc = "Add domain to list of blocked domains", + module = ?MODULE, + function = add_blocked_domain, + args = [{host, binary}, {domain, binary}], + result = {res, restuple}}, + #ejabberd_commands{name = remove_blocked_domain, + tags = [filter], + desc = "Remove domain from list of blocked domains", + module = ?MODULE, + function = remove_blocked_domain, + args = [{host, binary}, {domain, binary}], + result = {res, restuple}}]. for_all_hosts(F, A) -> - try lists:map( - fun(Host) -> - apply(F, [Host | A]) - end, get_spam_filter_hosts()) of - List -> - case lists:filter(fun({error, _}) -> true; (_) -> false end, List) of - [] -> hd(List); - Errors -> hd(Errors) - end - catch error:{badmatch, {error, _Reason} = Error} -> - Error + try lists:map(fun(Host) -> apply(F, [Host | A]) end, get_spam_filter_hosts()) of + List -> + case lists:filter(fun ({error, _}) -> + true; + (_) -> + false + end, + List) + of + [] -> + hd(List); + Errors -> + hd(Errors) + end + catch + error:{badmatch, {error, _Reason} = Error} -> + Error end. try_call_by_host(Host, Call) -> LServer = jid:nameprep(Host), Proc = get_proc_name(LServer), try gen_server:call(Proc, Call, ?COMMAND_TIMEOUT) of - Result -> - Result - catch exit:{noproc, _} -> - {error, "Not configured for " ++ binary_to_list(Host)}; - exit:{timeout, _} -> - {error, "Timeout while querying ejabberd"} + Result -> + Result + catch + exit:{noproc, _} -> + {error, "Not configured for " ++ binary_to_list(Host)}; + exit:{timeout, _} -> + {error, "Timeout while querying ejabberd"} end. -spec reload_spam_filter_files(binary()) -> ok | {error, string()}. @@ -972,25 +1061,32 @@ reload_spam_filter_files(<<"global">>) -> for_all_hosts(fun reload_spam_filter_files/1, []); reload_spam_filter_files(Host) -> LServer = jid:nameprep(Host), - Files = #{domains => gen_mod:get_module_opt(LServer, ?MODULE, spam_domains_file), - jid => gen_mod:get_module_opt(LServer, ?MODULE, spam_jids_file), - url => gen_mod:get_module_opt(LServer, ?MODULE, spam_urls_file)}, + Files = + #{domains => gen_mod:get_module_opt(LServer, ?MODULE, spam_domains_file), + jid => gen_mod:get_module_opt(LServer, ?MODULE, spam_jids_file), + url => gen_mod:get_module_opt(LServer, ?MODULE, spam_urls_file)}, case try_call_by_host(Host, {reload_files, Files}) of - {spam_filter, ok} -> - ok; - {spam_filter, {error, Txt}} -> - {error, binary_to_list(Txt)}; - {error, _R} = Error -> - Error + {spam_filter, ok} -> + ok; + {spam_filter, {error, Txt}} -> + {error, binary_to_list(Txt)}; + {error, _R} = Error -> + Error end. -spec get_blocked_domains(binary()) -> [binary()]. get_blocked_domains(Host) -> case try_call_by_host(Host, get_blocked_domains) of - {blocked_domains, BlockedDomains} -> - maps:keys(maps:filter(fun(_, false) -> false; (_, _) -> true end, BlockedDomains)); - {error, _R} = Error -> - Error + {blocked_domains, BlockedDomains} -> + maps:keys( + maps:filter(fun (_, false) -> + false; + (_, _) -> + true + end, + BlockedDomains)); + {error, _R} = Error -> + Error end. -spec add_blocked_domain(binary(), binary()) -> {ok, string()}. @@ -998,10 +1094,10 @@ add_blocked_domain(<<"global">>, Domain) -> for_all_hosts(fun add_blocked_domain/2, [Domain]); add_blocked_domain(Host, Domain) -> case try_call_by_host(Host, {add_blocked_domain, Domain}) of - {spam_filter, {Status, Txt}} -> - {Status, binary_to_list(Txt)}; - {error, _R} = Error -> - Error + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error end. -spec remove_blocked_domain(binary(), binary()) -> {ok, string()}. @@ -1009,21 +1105,19 @@ remove_blocked_domain(<<"global">>, Domain) -> for_all_hosts(fun remove_blocked_domain/2, [Domain]); remove_blocked_domain(Host, Domain) -> case try_call_by_host(Host, {remove_blocked_domain, Domain}) of - {spam_filter, {Status, Txt}} -> - {Status, binary_to_list(Txt)}; - {error, _R} = Error -> - Error + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error end. --spec get_spam_filter_cache(binary()) - -> [{binary(), integer()}] | {error, string()}. +-spec get_spam_filter_cache(binary()) -> [{binary(), integer()}] | {error, string()}. get_spam_filter_cache(Host) -> case try_call_by_host(Host, get_cache) of - {spam_filter, Cache} -> - [{jid:encode(JID), TS + erlang:time_offset(second)} || - {JID, TS} <- Cache]; - {error, _R} = Error -> - Error + {spam_filter, Cache} -> + [{jid:encode(JID), TS + erlang:time_offset(second)} || {JID, TS} <- Cache]; + {error, _R} = Error -> + Error end. -spec expire_spam_filter_cache(binary(), integer()) -> {ok | error, string()}. @@ -1031,27 +1125,31 @@ expire_spam_filter_cache(<<"global">>, Age) -> for_all_hosts(fun expire_spam_filter_cache/2, [Age]); expire_spam_filter_cache(Host, Age) -> case try_call_by_host(Host, {expire_cache, Age}) of - {spam_filter, {Status, Txt}} -> - {Status, binary_to_list(Txt)}; - {error, _R} = Error -> - Error + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error end. --spec add_to_spam_filter_cache(binary(), binary()) -> [{binary(), integer()}] | {error, string()}. +-spec add_to_spam_filter_cache(binary(), binary()) -> + [{binary(), integer()}] | {error, string()}. add_to_spam_filter_cache(<<"global">>, JID) -> for_all_hosts(fun add_to_spam_filter_cache/2, [JID]); add_to_spam_filter_cache(Host, EncJID) -> try jid:decode(EncJID) of - #jid{} = JID -> - LJID = jid:remove_resource(jid:tolower(JID)), - case try_call_by_host(Host, {add_to_cache, LJID}) of - {spam_filter, {Status, Txt}} -> - {Status, binary_to_list(Txt)}; - {error, _R} = Error -> - Error - end - catch _:{bad_jid, _} -> - {error, "Not a valid JID: " ++ binary_to_list(EncJID)} + #jid{} = JID -> + LJID = + jid:remove_resource( + jid:tolower(JID)), + case try_call_by_host(Host, {add_to_cache, LJID}) of + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end + catch + _:{bad_jid, _} -> + {error, "Not a valid JID: " ++ binary_to_list(EncJID)} end. -spec drop_from_spam_filter_cache(binary(), binary()) -> {ok | error, string()}. @@ -1059,14 +1157,17 @@ drop_from_spam_filter_cache(<<"global">>, JID) -> for_all_hosts(fun drop_from_spam_filter_cache/2, [JID]); drop_from_spam_filter_cache(Host, EncJID) -> try jid:decode(EncJID) of - #jid{} = JID -> - LJID = jid:remove_resource(jid:tolower(JID)), - case try_call_by_host(Host, {drop_from_cache, LJID}) of - {spam_filter, {Status, Txt}} -> - {Status, binary_to_list(Txt)}; - {error, _R} = Error -> - Error - end - catch _:{bad_jid, _} -> - {error, "Not a valid JID: " ++ binary_to_list(EncJID)} + #jid{} = JID -> + LJID = + jid:remove_resource( + jid:tolower(JID)), + case try_call_by_host(Host, {drop_from_cache, LJID}) of + {spam_filter, {Status, Txt}} -> + {Status, binary_to_list(Txt)}; + {error, _R} = Error -> + Error + end + catch + _:{bad_jid, _} -> + {error, "Not a valid JID: " ++ binary_to_list(EncJID)} end. diff --git a/src/mod_antispam_rtbl.erl b/src/mod_antispam_rtbl.erl index d977aa0c6..93b346631 100644 --- a/src/mod_antispam_rtbl.erl +++ b/src/mod_antispam_rtbl.erl @@ -38,11 +38,15 @@ subscribe/3, unsubscribe/3]). +%% @format-begin + subscribe(RTBLHost, RTBLDomainsNode, From) -> FromJID = service_jid(From), - SubIQ = #iq{type = set, to = jid:make(RTBLHost), from = FromJID, - sub_els = [ - #pubsub{subscribe = #ps_subscribe{jid = FromJID, node = RTBLDomainsNode}}]}, + SubIQ = + #iq{type = set, + to = jid:make(RTBLHost), + from = FromJID, + sub_els = [#pubsub{subscribe = #ps_subscribe{jid = FromJID, node = RTBLDomainsNode}}]}, ?DEBUG("Sending subscription request:~n~p", [xmpp:encode(SubIQ)]), ejabberd_router:route_iq(SubIQ, subscribe_result, self()). @@ -51,19 +55,22 @@ unsubscribe(none, _PSNode, _From) -> ok; unsubscribe(RTBLHost, RTBLDomainsNode, From) -> FromJID = jid:make(From), - SubIQ = #iq{type = set, to = jid:make(RTBLHost), from = FromJID, - sub_els = [ - #pubsub{unsubscribe = #ps_unsubscribe{jid = FromJID, node = RTBLDomainsNode}}]}, + SubIQ = + #iq{type = set, + to = jid:make(RTBLHost), + from = FromJID, + sub_els = + [#pubsub{unsubscribe = #ps_unsubscribe{jid = FromJID, node = RTBLDomainsNode}}]}, ejabberd_router:route_iq(SubIQ, unsubscribe_result, self()). -spec request_blocked_domains(binary() | none, binary(), binary()) -> ok. request_blocked_domains(none, _PSNode, _From) -> ok; request_blocked_domains(RTBLHost, RTBLDomainsNode, From) -> - IQ = #iq{type = get, from = jid:make(From), - to = jid:make(RTBLHost), - sub_els = [ - #pubsub{items = #ps_items{node = RTBLDomainsNode}}]}, + IQ = #iq{type = get, + from = jid:make(From), + to = jid:make(RTBLHost), + sub_els = [#pubsub{items = #ps_items{node = RTBLDomainsNode}}]}, ?DEBUG("Requesting RTBL blocked domains from ~s:~n~p", [RTBLHost, xmpp:encode(IQ)]), ejabberd_router:route_iq(IQ, blocked_domains, self()). @@ -72,31 +79,35 @@ parse_blocked_domains(#iq{to = #jid{lserver = LServer}, type = result} = IQ) -> ?DEBUG("parsing iq-result items: ~p", [IQ]), RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), case xmpp:get_subtag(IQ, #pubsub{}) of - #pubsub{items = #ps_items{node = RTBLDomainsNode, items = Items}} -> - ?DEBUG("Got items:~n~p", [Items]), - parse_items(Items); - _ -> - undefined + #pubsub{items = #ps_items{node = RTBLDomainsNode, items = Items}} -> + ?DEBUG("Got items:~n~p", [Items]), + parse_items(Items); + _ -> + undefined end. -spec parse_pubsub_event(stanza()) -> #{binary() => any()}. parse_pubsub_event(#message{to = #jid{lserver = LServer}} = Msg) -> RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), case xmpp:get_subtag(Msg, #ps_event{}) of - #ps_event{items = #ps_items{node = RTBLDomainsNode, items = Items, retract = RetractIds}} -> - maps:merge(retract_items(RetractIds), parse_items(Items)); - Other -> - ?WARNING_MSG("Couldn't extract items: ~p", [Other]), - #{} + #ps_event{items = + #ps_items{node = RTBLDomainsNode, + items = Items, + retract = RetractIds}} -> + maps:merge(retract_items(RetractIds), parse_items(Items)); + Other -> + ?WARNING_MSG("Couldn't extract items: ~p", [Other]), + #{} end. -spec parse_items([ps_item()]) -> #{binary() => any()}. parse_items(Items) -> - lists:foldl( - fun(#ps_item{id = ID}, Acc) -> - %% TODO extract meta/extra instructions - maps:put(ID, true, Acc) - end, #{}, Items). + lists:foldl(fun(#ps_item{id = ID}, Acc) -> + %% TODO extract meta/extra instructions + maps:put(ID, true, Acc) + end, + #{}, + Items). -spec retract_items([binary()]) -> #{binary() => false}. retract_items(Ids) -> @@ -112,20 +123,22 @@ service_jid(Host) -> -spec pubsub_event_handler(stanza()) -> drop | stanza(). pubsub_event_handler(#message{from = FromJid, - to = #jid{lserver = LServer, - lresource = <>}} = Msg) -> + to = + #jid{lserver = LServer, + lresource = <>}} = + Msg) -> ?DEBUG("Got RTBL message:~n~p", [Msg]), From = jid:encode(FromJid), case gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_host) of - From -> - ParsedItems = parse_pubsub_event(Msg), - Proc = gen_mod:get_module_proc(LServer, ?SERVICE_MODULE), - gen_server:cast(Proc, {update_blocked_domains, ParsedItems}), - %% FIXME what's the difference between `{drop, ...}` and `{stop, {drop, ...}}`? - drop; - _Other -> - ?INFO_MSG("Got unexpected message from ~s to rtbl resource:~n~p", [From, Msg]), - Msg + From -> + ParsedItems = parse_pubsub_event(Msg), + Proc = gen_mod:get_module_proc(LServer, ?SERVICE_MODULE), + gen_server:cast(Proc, {update_blocked_domains, ParsedItems}), + %% FIXME what's the difference between `{drop, ...}` and `{stop, {drop, ...}}`? + drop; + _Other -> + ?INFO_MSG("Got unexpected message from ~s to rtbl resource:~n~p", [From, Msg]), + Msg end; pubsub_event_handler(Acc) -> ?DEBUG("unexpected something on pubsub_event_handler: ~p", [Acc]), diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index d7ce196c0..6d94b8750 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -31,6 +31,8 @@ my_muc_jid/1, get_features/2, set_opt/3]). -include("suite.hrl"). +%% @format-begin + %%%=================================================================== %%% API %%%=================================================================== @@ -38,35 +40,51 @@ %%% Single tests %%%=================================================================== single_cases() -> - {antispam_single, [sequence], - [single_test(spam_files), - single_test(blocked_domains), - single_test(jid_cache), - single_test(rtbl_domains)]}. + {antispam_single, + [sequence], + [single_test(spam_files), + single_test(blocked_domains), + single_test(jid_cache), + single_test(rtbl_domains)]}. spam_files(Config) -> Host = ?config(server, Config), To = my_jid(Config), SpamJID = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>), - SpamJIDMsg = #message{from = SpamJID, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + SpamJIDMsg = + #message{from = SpamJID, + to = To, + type = chat, + body = [#text{data = <<"hello world">>}]}, is_spam(SpamJIDMsg), Spammer = jid:make(<<"spammer">>, <<"localhost">>, <<"spam_client">>), - NoSpamMsg = #message{from = Spammer, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + NoSpamMsg = + #message{from = Spammer, + to = To, + type = chat, + body = [#text{data = <<"hello world">>}]}, is_not_spam(NoSpamMsg), - SpamMsg = #message{from = Spammer, to = To, type = chat, body = [#text{data = <<"hello world\nhttps://spam.domain.url">>}]}, + SpamMsg = + #message{from = Spammer, + to = To, + type = chat, + body = [#text{data = <<"hello world\nhttps://spam.domain.url">>}]}, is_spam(SpamMsg), %% now check this mischief is in jid_cache is_spam(NoSpamMsg), mod_antispam:drop_from_spam_filter_cache(Host, jid:to_string(Spammer)), is_not_spam(NoSpamMsg), - ?retry(100, 10, - ?match(true, (has_spam_domain(<<"spam_domain.org">>))(Host))), + ?retry(100, 10, ?match(true, (has_spam_domain(<<"spam_domain.org">>))(Host))), SpamDomain = jid:make(<<"spammer">>, <<"spam_domain.org">>, <<"spam_client">>), - SpamDomainMsg = #message{from = SpamDomain, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + SpamDomainMsg = + #message{from = SpamDomain, + to = To, + type = chat, + body = [#text{data = <<"hello world">>}]}, is_spam(SpamDomainMsg), ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)), ?match([], mod_antispam:get_blocked_domains(Host)), @@ -78,9 +96,12 @@ blocked_domains(Config) -> ?match([], mod_antispam:get_blocked_domains(Host)), SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), To = my_jid(Config), - Msg = #message{from = SpamFrom, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + Msg = #message{from = SpamFrom, + to = To, + type = chat, + body = [#text{data = <<"hello world">>}]}, is_not_spam(Msg), - ?match({ok, _}, mod_antispam:add_blocked_domain(<<"global">>, <<"spam.domain">>)), + ?match({ok, _}, mod_antispam:add_blocked_domain(<<"global">>, <<"spam.domain">>)), is_spam(Msg), Vhosts = [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, mod_antispam)], NumVhosts = length(Vhosts), @@ -102,7 +123,10 @@ jid_cache(Config) -> Host = ?config(server, Config), SpamFrom = jid:make(<<"spammer">>, Host, <<"spam_client">>), To = my_jid(Config), - Msg = #message{from = SpamFrom, to = To, type = chat, body = [#text{data = <<"hello world">>}]}, + Msg = #message{from = SpamFrom, + to = To, + type = chat, + body = [#text{data = <<"hello world">>}]}, is_not_spam(Msg), mod_antispam:add_to_spam_filter_cache(Host, jid:to_string(SpamFrom)), is_spam(Msg), @@ -112,25 +136,45 @@ jid_cache(Config) -> rtbl_domains(Config) -> Host = ?config(server, Config), - RTBLHost = jid:to_string(suite:pubsub_jid(Config)), + RTBLHost = + jid:to_string( + suite:pubsub_jid(Config)), RTBLDomainsNode = <<"spam_source_domains">>, OldOpts = gen_mod:get_module_opts(Host, mod_antispam), - NewOpts = maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}), + NewOpts = + maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}), Owner = jid:make(?config(user, Config), ?config(server, Config), <<>>), - {result, _} = mod_pubsub:create_node(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"flat">>), - {result, _} = mod_pubsub:publish_item(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"spam.source.domain">>, - [xmpp:encode(#ps_item{id = <<"spam.source.domain">>, sub_els = []})]), + {result, _} = + mod_pubsub:create_node(RTBLHost, + ?config(server, Config), + RTBLDomainsNode, + Owner, + <<"flat">>), + {result, _} = + mod_pubsub:publish_item(RTBLHost, + ?config(server, Config), + RTBLDomainsNode, + Owner, + <<"spam.source.domain">>, + [xmpp:encode(#ps_item{id = <<"spam.source.domain">>, + sub_els = []})]), mod_antispam:reload(Host, OldOpts, NewOpts), ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)), - ?retry(100, 10, + ?retry(100, + 10, ?match([<<"spam.source.domain">>], mod_antispam:get_blocked_domains(Host))), - {result, _} = mod_pubsub:publish_item(RTBLHost, ?config(server, Config), RTBLDomainsNode, Owner, <<"spam.source.another">>, - [xmpp:encode(#ps_item{id = <<"spam.source.another">>, sub_els = []})]), - ?retry(100, 10, - ?match(true, (has_spam_domain(<<"spam.source.another">>))(Host))), - {result, _} = mod_pubsub:delete_item(RTBLHost, RTBLDomainsNode, Owner, <<"spam.source.another">>, true), - ?retry(100, 10, - ?match(false, (has_spam_domain(<<"spam.source.another">>))(Host))), + {result, _} = + mod_pubsub:publish_item(RTBLHost, + ?config(server, Config), + RTBLDomainsNode, + Owner, + <<"spam.source.another">>, + [xmpp:encode(#ps_item{id = <<"spam.source.another">>, + sub_els = []})]), + ?retry(100, 10, ?match(true, (has_spam_domain(<<"spam.source.another">>))(Host))), + {result, _} = + mod_pubsub:delete_item(RTBLHost, RTBLDomainsNode, Owner, <<"spam.source.another">>, true), + ?retry(100, 10, ?match(false, (has_spam_domain(<<"spam.source.another">>))(Host))), {result, _} = mod_pubsub:delete_node(RTBLHost, RTBLDomainsNode, Owner), disconnect(Config). @@ -141,9 +185,7 @@ single_test(T) -> list_to_atom("antispam_" ++ atom_to_list(T)). has_spam_domain(Domain) -> - fun(Host) -> - lists:member(Domain, mod_antispam:get_blocked_domains(Host)) - end. + fun(Host) -> lists:member(Domain, mod_antispam:get_blocked_domains(Host)) end. is_not_spam(Msg) -> ?match({Msg, undefined}, mod_antispam:s2s_receive_packet({Msg, undefined})). From ee46333def6bfcec25ae255d144e03f0d3c23030 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 9 Jun 2025 16:26:00 +0200 Subject: [PATCH 06/27] add make target test- Eg. invoke common test for specific test group only like $ CT_BACKEND=mnesia,redis make test-antispam_single --- Makefile.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Makefile.in b/Makefile.in index c10ff390d..cf7480702 100644 --- a/Makefile.in +++ b/Makefile.in @@ -661,6 +661,17 @@ test: @cd priv && ln -sf ../sql $(REBAR) $(SKIPDEPS) ct +.PHONY: test-% +define test-group-target +test-$1: + $(REBAR) $(SKIPDEPS) ct --suite=test/ejabberd_SUITE --group=$1 +endef + +ifneq ($(filter test-%,$(MAKECMDGOALS)),) +group_to_test := $(patsubst test-%,%,$(filter test-%,$(MAKECMDGOALS))) +$(eval $(call test-group-target,$(group_to_test))) +endif + test-eunit: $(REBAR) $(SKIPDEPS) eunit --verbose @@ -711,6 +722,7 @@ help: @echo " hooks Run hooks validator" @echo " test Run Common Tests suite [rebar3]" @echo " test-eunit Run EUnit suite [rebar3]" + @echo " test- Run Common Test suite for specific group only [rebar3]" @echo " xref Run cross reference analysis [rebar3]" #. From b607d95a93156420e866aef2c8ef6e6d8ebd4a80 Mon Sep 17 00:00:00 2001 From: Badlop Date: Fri, 6 Jun 2025 17:38:13 +0200 Subject: [PATCH 07/27] Refactorize each individual test case in individual functions --- test/antispam_tests.erl | 142 +++++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 52 deletions(-) diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index 6d94b8750..7c3b053b1 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -20,6 +20,7 @@ %%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. %%% %%%---------------------------------------------------------------------- + -module(antispam_tests). -compile(export_all). @@ -39,59 +40,60 @@ %%%=================================================================== %%% Single tests %%%=================================================================== + single_cases() -> {antispam_single, [sequence], - [single_test(spam_files), - single_test(blocked_domains), + [single_test(block_by_jid), + single_test(block_by_url), + single_test(blocked_jid_is_cached), + single_test(uncache_blocked_jid), + single_test(check_blocked_domain), + single_test(unblock_domain), + single_test(empty_domain_list), + single_test(block_domain_globally), + single_test(check_domain_blocked_globally), + single_test(unblock_domain_in_vhost), + single_test(unblock_domain_globally), + single_test(block_domain_in_vhost), + single_test(unblock_domain_in_vhost2), single_test(jid_cache), single_test(rtbl_domains)]}. -spam_files(Config) -> - Host = ?config(server, Config), +%%%=================================================================== + +block_by_jid(Config) -> + is_spam(message_hello(<<"spammer_jid">>, <<"localhost">>, Config)). + +block_by_url(Config) -> + From = jid:make(<<"spammer">>, <<"localhost">>, <<"spam_client">>), To = my_jid(Config), + is_not_spam(message_hello(<<"spammer">>, <<"localhost">>, Config)), + is_spam(message(From, To, <<"hello world\nhttps://spam.domain.url">>)). - SpamJID = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>), - SpamJIDMsg = - #message{from = SpamJID, - to = To, - type = chat, - body = [#text{data = <<"hello world">>}]}, - is_spam(SpamJIDMsg), +blocked_jid_is_cached(Config) -> + is_spam(message_hello(<<"spammer">>, <<"localhost">>, Config)). - Spammer = jid:make(<<"spammer">>, <<"localhost">>, <<"spam_client">>), - NoSpamMsg = - #message{from = Spammer, - to = To, - type = chat, - body = [#text{data = <<"hello world">>}]}, - is_not_spam(NoSpamMsg), - SpamMsg = - #message{from = Spammer, - to = To, - type = chat, - body = [#text{data = <<"hello world\nhttps://spam.domain.url">>}]}, - is_spam(SpamMsg), - %% now check this mischief is in jid_cache - is_spam(NoSpamMsg), +uncache_blocked_jid(Config) -> + Host = ?config(server, Config), + Spammer = jid:make(<<"spammer">>, <<"localhost">>, <<"">>), mod_antispam:drop_from_spam_filter_cache(Host, jid:to_string(Spammer)), - is_not_spam(NoSpamMsg), + is_not_spam(message_hello(<<"spammer">>, <<"localhost">>, Config)). +check_blocked_domain(Config) -> + Host = ?config(server, Config), ?retry(100, 10, ?match(true, (has_spam_domain(<<"spam_domain.org">>))(Host))), + is_spam(message_hello(<<"other_spammer">>, <<"spam_domain.org">>, Config)). - SpamDomain = jid:make(<<"spammer">>, <<"spam_domain.org">>, <<"spam_client">>), - SpamDomainMsg = - #message{from = SpamDomain, - to = To, - type = chat, - body = [#text{data = <<"hello world">>}]}, - is_spam(SpamDomainMsg), +unblock_domain(Config) -> + Host = ?config(server, Config), ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam_domain.org">>)), ?match([], mod_antispam:get_blocked_domains(Host)), - is_not_spam(SpamDomainMsg), - disconnect(Config). + is_not_spam(message_hello(<<"spammer">>, <<"spam_domain.org">>, Config)). -blocked_domains(Config) -> +%%%=================================================================== + +empty_domain_list(Config) -> Host = ?config(server, Config), ?match([], mod_antispam:get_blocked_domains(Host)), SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), @@ -100,40 +102,65 @@ blocked_domains(Config) -> to = To, type = chat, body = [#text{data = <<"hello world">>}]}, - is_not_spam(Msg), + is_not_spam(Msg). + +block_domain_globally(Config) -> ?match({ok, _}, mod_antispam:add_blocked_domain(<<"global">>, <<"spam.domain">>)), - is_spam(Msg), + SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), + To = my_jid(Config), + is_spam(message(SpamFrom, To, <<"hello world">>)). + +check_domain_blocked_globally(_Config) -> Vhosts = [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, mod_antispam)], NumVhosts = length(Vhosts), - ?match(NumVhosts, length(lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts))), + ?match(NumVhosts, length(lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts))). + +unblock_domain_in_vhost(Config) -> + Host = ?config(server, Config), ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam.domain">>)), ?match([], mod_antispam:get_blocked_domains(Host)), - is_not_spam(Msg), + SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), + To = my_jid(Config), + is_not_spam(message(SpamFrom, To, <<"hello world">>)). + +unblock_domain_globally(_Config) -> + Vhosts = [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, mod_antispam)], + NumVhosts = length(Vhosts), ?match(NumVhosts, length(lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts)) + 1), ?match({ok, _}, mod_antispam:remove_blocked_domain(<<"global">>, <<"spam.domain">>)), - ?match([], lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts)), + ?match([], lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts)). + +block_domain_in_vhost(Config) -> + Host = ?config(server, Config), + Vhosts = [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, mod_antispam)], ?match({ok, _}, mod_antispam:add_blocked_domain(Host, <<"spam.domain">>)), ?match([Host], lists:filter(has_spam_domain(<<"spam.domain">>), Vhosts)), - is_spam(Msg), + SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), + To = my_jid(Config), + is_spam(message(SpamFrom, To, <<"hello world">>)). + +unblock_domain_in_vhost2(Config) -> + Host = ?config(server, Config), ?match({ok, _}, mod_antispam:remove_blocked_domain(Host, <<"spam.domain">>)), - is_not_spam(Msg), + SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), + To = my_jid(Config), + is_not_spam(message(SpamFrom, To, <<"hello world">>)), disconnect(Config). +%%%=================================================================== + jid_cache(Config) -> Host = ?config(server, Config), SpamFrom = jid:make(<<"spammer">>, Host, <<"spam_client">>), - To = my_jid(Config), - Msg = #message{from = SpamFrom, - to = To, - type = chat, - body = [#text{data = <<"hello world">>}]}, - is_not_spam(Msg), + is_not_spam(message_hello(<<"spammer">>, Host, Config)), mod_antispam:add_to_spam_filter_cache(Host, jid:to_string(SpamFrom)), - is_spam(Msg), + is_spam(message_hello(<<"spammer">>, Host, Config)), mod_antispam:drop_from_spam_filter_cache(Host, jid:to_string(SpamFrom)), - is_not_spam(Msg), + is_not_spam(message_hello(<<"spammer">>, Host, Config)), disconnect(Config). +%%%=================================================================== + rtbl_domains(Config) -> Host = ?config(server, Config), RTBLHost = @@ -192,3 +219,14 @@ is_not_spam(Msg) -> is_spam(Spam) -> ?match({stop, {drop, undefined}}, mod_antispam:s2s_receive_packet({Spam, undefined})). + +message_hello(Username, Host, Config) -> + SpamFrom = jid:make(Username, Host, <<"spam_client">>), + To = my_jid(Config), + message(SpamFrom, To, <<"hello world">>). + +message(From, To, BodyText) -> + #message{from = From, + to = To, + type = chat, + body = [#text{data = BodyText}]}. From 6122a525d24b379b68f3cff863f10bbbb52016a7 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 9 Jun 2025 16:31:26 +0200 Subject: [PATCH 08/27] mod_antispam: fix config types --- src/mod_antispam.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index c3489aefa..664e5d67f 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -138,10 +138,11 @@ mod_opt_type(spam_domains_file) -> econf:either( econf:enum([none]), econf:file()); mod_opt_type(whitelist_domains_file) -> - econf:either(none, econf:binary()); + econf:either( + econf:enum([none]), econf:file()); mod_opt_type(spam_dump_file) -> econf:either( - econf:enum([none]), econf:binary()); + econf:enum([none]), econf:file()); mod_opt_type(spam_jids_file) -> econf:either( econf:enum([none]), econf:file()); From ea19e4bc7fcacb261d4eb4b1e92533af055875f7 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 9 Jun 2025 16:35:10 +0200 Subject: [PATCH 09/27] mod_antispam: remove unnecessary check in test this was left over from debugging issues with fixtures --- test/antispam_tests.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index 7c3b053b1..da69bd630 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -81,8 +81,6 @@ uncache_blocked_jid(Config) -> is_not_spam(message_hello(<<"spammer">>, <<"localhost">>, Config)). check_blocked_domain(Config) -> - Host = ?config(server, Config), - ?retry(100, 10, ?match(true, (has_spam_domain(<<"spam_domain.org">>))(Host))), is_spam(message_hello(<<"other_spammer">>, <<"spam_domain.org">>, Config)). unblock_domain(Config) -> From 7a6e4098797b24fc6e512e68d5e32c07efdfb4ef Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 9 Jun 2025 16:38:52 +0200 Subject: [PATCH 10/27] mod_antispam: use message/3 in test --- test/antispam_tests.erl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index da69bd630..f6f589c0d 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -96,10 +96,7 @@ empty_domain_list(Config) -> ?match([], mod_antispam:get_blocked_domains(Host)), SpamFrom = jid:make(<<"spammer">>, <<"spam.domain">>, <<"spam_client">>), To = my_jid(Config), - Msg = #message{from = SpamFrom, - to = To, - type = chat, - body = [#text{data = <<"hello world">>}]}, + Msg = message(SpamFrom, To, <<"hello world">>), is_not_spam(Msg). block_domain_globally(Config) -> From 10ec128b945ad52183ba239d488c982e1684d8e2 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 9 Jun 2025 16:55:31 +0200 Subject: [PATCH 11/27] mod_antispam: test whitelisted domain --- test/antispam_tests.erl | 43 ++++++++++++++++++- test/ejabberd_SUITE_data/ejabberd.mnesia.yml | 1 + test/ejabberd_SUITE_data/ejabberd.redis.yml | 1 + .../ejabberd_SUITE_data/whitelist_domains.txt | 1 + test/suite.erl | 1 + 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 test/ejabberd_SUITE_data/whitelist_domains.txt diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index f6f589c0d..d56ac53cd 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -58,7 +58,8 @@ single_cases() -> single_test(block_domain_in_vhost), single_test(unblock_domain_in_vhost2), single_test(jid_cache), - single_test(rtbl_domains)]}. + single_test(rtbl_domains), + single_test(rtbl_domains_whitelisted)]}. %%%=================================================================== @@ -200,6 +201,46 @@ rtbl_domains(Config) -> {result, _} = mod_pubsub:delete_node(RTBLHost, RTBLDomainsNode, Owner), disconnect(Config). +rtbl_domains_whitelisted(Config) -> + Host = ?config(server, Config), + RTBLHost = + jid:to_string( + suite:pubsub_jid(Config)), + RTBLDomainsNode = <<"spam_source_domains">>, + OldOpts = gen_mod:get_module_opts(Host, mod_antispam), + NewOpts = + maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}), + Owner = jid:make(?config(user, Config), ?config(server, Config), <<>>), + {result, _} = + mod_pubsub:create_node(RTBLHost, + ?config(server, Config), + RTBLDomainsNode, + Owner, + <<"flat">>), + {result, _} = + mod_pubsub:publish_item(RTBLHost, + ?config(server, Config), + RTBLDomainsNode, + Owner, + <<"whitelisted.domain">>, + [xmpp:encode(#ps_item{id = <<"whitelisted.domain">>, + sub_els = []})]), + mod_antispam:reload(Host, OldOpts, NewOpts), + {result, _} = + mod_pubsub:publish_item(RTBLHost, + ?config(server, Config), + RTBLDomainsNode, + Owner, + <<"yetanother.domain">>, + [xmpp:encode(#ps_item{id = <<"yetanother.domain">>, + sub_els = []})]), + ?retry(100, 10, ?match(true, (has_spam_domain(<<"yetanother.domain">>))(Host))), + %% we assume that the previous "whitelisted.domain" pubsub item has been consumed by now, so we + %% can check that it doesn't exist + ?match(false, (has_spam_domain(<<"whitelisted.domain">>))(Host)), + {result, _} = mod_pubsub:delete_node(RTBLHost, RTBLDomainsNode, Owner), + disconnect(Config). + %%%=================================================================== %%% Internal functions %%%=================================================================== diff --git a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml index c8585e6e4..80f4da81c 100644 --- a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml +++ b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml @@ -11,6 +11,7 @@ define_macro: spam_jids_file: spam_jids.txt spam_domains_file: spam_domains.txt spam_urls_file: spam_urls.txt + whitelist_domains_file: whitelist_domains.txt mod_blocking: [] mod_caps: db_type: internal diff --git a/test/ejabberd_SUITE_data/ejabberd.redis.yml b/test/ejabberd_SUITE_data/ejabberd.redis.yml index cdbc905bd..14737b17c 100644 --- a/test/ejabberd_SUITE_data/ejabberd.redis.yml +++ b/test/ejabberd_SUITE_data/ejabberd.redis.yml @@ -12,6 +12,7 @@ define_macro: spam_jids_file: spam_jids.txt spam_domains_file: spam_domains.txt spam_urls_file: spam_urls.txt + whitelist_domains_file: whitelist_domains.txt mod_blocking: [] mod_caps: db_type: internal diff --git a/test/ejabberd_SUITE_data/whitelist_domains.txt b/test/ejabberd_SUITE_data/whitelist_domains.txt new file mode 100644 index 000000000..b953fb7f6 --- /dev/null +++ b/test/ejabberd_SUITE_data/whitelist_domains.txt @@ -0,0 +1 @@ +whitelisted.domain diff --git a/test/suite.erl b/test/suite.erl index 62b442580..7bc27e913 100644 --- a/test/suite.erl +++ b/test/suite.erl @@ -54,6 +54,7 @@ init_config(Config) -> copy_file(Config, "spam_jids.txt"), copy_file(Config, "spam_urls.txt"), copy_file(Config, "spam_domains.txt"), + copy_file(Config, "whitelist_domains.txt"), {ok, MacrosContentTpl} = file:read_file(MacrosPathTpl), Password = <<"password!@#$%^&*()'\"`~<>+-/;:_=[]{}|\\">>, Backends = get_config_backends(), From bae345b92b57a6f61ec1654b90ddc9a75b0184f1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 17 Jun 2025 12:24:02 +0200 Subject: [PATCH 12/27] mod_antispam: test dump file --- test/antispam_tests.erl | 21 +++++++++++++++++++- test/ejabberd_SUITE_data/ejabberd.mnesia.yml | 1 + test/ejabberd_SUITE_data/ejabberd.redis.yml | 1 + test/suite.erl | 1 + 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index d56ac53cd..0eb8d4041 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -59,7 +59,8 @@ single_cases() -> single_test(unblock_domain_in_vhost2), single_test(jid_cache), single_test(rtbl_domains), - single_test(rtbl_domains_whitelisted)]}. + single_test(rtbl_domains_whitelisted), + single_test(spam_dump_file)]}. %%%=================================================================== @@ -241,6 +242,20 @@ rtbl_domains_whitelisted(Config) -> {result, _} = mod_pubsub:delete_node(RTBLHost, RTBLDomainsNode, Owner), disconnect(Config). +%%%=================================================================== + +spam_dump_file(Config) -> + {ok, CWD} = file:get_cwd(), + Filename = filename:join([CWD, "spam.log"]), + ?retry(100, 10, + ?match(true, size(get_bytes(Filename)) > 0)), + From = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>), + To = my_jid(Config), + is_spam(message(From, To, <<"A very specific spam message">>)), + ?retry(100, 100, + ?match({match, _}, + re:run(get_bytes(Filename), <<"A very specific spam message">>))). + %%%=================================================================== %%% Internal functions %%%=================================================================== @@ -266,3 +281,7 @@ message(From, To, BodyText) -> to = To, type = chat, body = [#text{data = BodyText}]}. + +get_bytes(Filename) -> + {ok, Bytes} = file:read_file(Filename), + Bytes. diff --git a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml index 80f4da81c..ae78645ae 100644 --- a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml +++ b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml @@ -12,6 +12,7 @@ define_macro: spam_domains_file: spam_domains.txt spam_urls_file: spam_urls.txt whitelist_domains_file: whitelist_domains.txt + spam_dump_file: spam.log mod_blocking: [] mod_caps: db_type: internal diff --git a/test/ejabberd_SUITE_data/ejabberd.redis.yml b/test/ejabberd_SUITE_data/ejabberd.redis.yml index 14737b17c..8ec927e86 100644 --- a/test/ejabberd_SUITE_data/ejabberd.redis.yml +++ b/test/ejabberd_SUITE_data/ejabberd.redis.yml @@ -13,6 +13,7 @@ define_macro: spam_domains_file: spam_domains.txt spam_urls_file: spam_urls.txt whitelist_domains_file: whitelist_domains.txt + spam_dump_file: spam.log mod_blocking: [] mod_caps: db_type: internal diff --git a/test/suite.erl b/test/suite.erl index 7bc27e913..5fbd70463 100644 --- a/test/suite.erl +++ b/test/suite.erl @@ -55,6 +55,7 @@ init_config(Config) -> copy_file(Config, "spam_urls.txt"), copy_file(Config, "spam_domains.txt"), copy_file(Config, "whitelist_domains.txt"), + file:write_file(filename:join([CWD, "spam.log"]), []), {ok, MacrosContentTpl} = file:read_file(MacrosPathTpl), Password = <<"password!@#$%^&*()'\"`~<>+-/;:_=[]{}|\\">>, Backends = get_config_backends(), From 149b715b4f31ec9c22aba7445010e3b8c21ab88a Mon Sep 17 00:00:00 2001 From: Badlop Date: Tue, 10 Jun 2025 13:52:59 +0200 Subject: [PATCH 13/27] New predefined keyword: LOG_PATH --- src/ejabberd_config.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ejabberd_config.erl b/src/ejabberd_config.erl index 3a556d7f6..89f89504d 100644 --- a/src/ejabberd_config.erl +++ b/src/ejabberd_config.erl @@ -504,8 +504,12 @@ get_predefined_keywords(Host) -> [{<<"HOST">>, Host}] end, Home = misc:get_home(), + LogDirPath = + iolist_to_binary(filename:dirname( + ejabberd_logger:get_log_path())), HostList ++ [{<<"HOME">>, list_to_binary(Home)}, + {<<"LOG_PATH">>, LogDirPath}, {<<"SEMVER">>, ejabberd_option:version()}, {<<"VERSION">>, misc:semver_to_xxyy( From 85f05192c821782d5748bd0f334518a4e3614aa4 Mon Sep 17 00:00:00 2001 From: Badlop Date: Tue, 10 Jun 2025 16:56:50 +0200 Subject: [PATCH 14/27] Move spam_dump_file implementation to a submodule --- src/mod_antispam.erl | 170 ++++++++++------------------------ src/mod_antispam_dump.erl | 189 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 120 deletions(-) create mode 100644 src/mod_antispam_dump.erl diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index 664e5d67f..8b161b558 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -24,6 +24,8 @@ %%% %%%---------------------------------------------------------------------- +%%| definitions + -module(mod_antispam). -author('holger@zedat.fu-berlin.de'). -author('stefan@strigler.de'). @@ -51,8 +53,7 @@ %% ejabberd_hooks callbacks. -export([s2s_in_handle_info/2, s2s_receive_packet/1, - sm_receive_packet/1, - reopen_log/0]). + sm_receive_packet/1]). %% ejabberd_commands callbacks. -export([add_blocked_domain/2, @@ -67,11 +68,12 @@ -include("ejabberd_commands.hrl"). -include("logger.hrl"). +-include("translate.hrl"). -include_lib("xmpp/include/xmpp.hrl"). -type url() :: binary(). --type filename() :: binary() | none. +-type filename() :: binary() | none | false. -type jid_set() :: sets:set(ljid()). -type url_set() :: sets:set(url()). -type s2s_in_state() :: ejabberd_s2s_in:state(). @@ -101,8 +103,8 @@ %% @format-begin %%-------------------------------------------------------------------- -%% gen_mod callbacks. -%%-------------------------------------------------------------------- +%%| gen_mod callbacks + -spec start(binary(), gen_mod:opts()) -> ok | {error, any()}. start(Host, Opts) -> case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of @@ -142,7 +144,7 @@ mod_opt_type(whitelist_domains_file) -> econf:enum([none]), econf:file()); mod_opt_type(spam_dump_file) -> econf:either( - econf:enum([none]), econf:file()); + econf:bool(), econf:file(write)); mod_opt_type(spam_jids_file) -> econf:either( econf:enum([none]), econf:file()); @@ -163,7 +165,7 @@ mod_opt_type(rtbl_domains_node) -> -spec mod_options(binary()) -> [{atom(), any()}]. mod_options(_Host) -> [{spam_domains_file, none}, - {spam_dump_file, none}, + {spam_dump_file, false}, {spam_jids_file, none}, {spam_urls_file, none}, {whitelist_domains_file, none}, @@ -173,15 +175,28 @@ mod_options(_Host) -> {rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}]. mod_doc() -> - #{}. + #{desc => ?T("Reads from text file and RTBL, filters stanzas and writes dump file."), + note => "added in 25.xx", + opts => + [{spam_dump_file, + #{value => ?T("false | true | Path"), + desc => + ?T("Path to the file to store blocked messages. " + "Use an absolute path, or the '@LOG_PATH@' macro to store logs " + "in the same place that the other ejabberd log files. " + "If set to 'false', does not dump stanzas, this is the default. " + "If set to 'true', it stores in '\"@LOG_PATH@/spam_dump_@HOST@.log\"'.")}}], + example => + ["modules:", + " mod_antispam:", + " spam_dump_file: \"@LOG_PATH@/spam/host-@HOST@.log\""]}. %%-------------------------------------------------------------------- -%% gen_server callbacks. -%%-------------------------------------------------------------------- +%%| gen_server callbacks + -spec init(list()) -> {ok, state()} | {stop, term()}. init([Host, Opts]) -> process_flag(trap_exit, true), - DumpFile = expand_host(gen_mod:get_opt(spam_dump_file, Opts), Host), Files = #{domains => gen_mod:get_opt(spam_domains_file, Opts), jid => gen_mod:get_opt(spam_jids_file, Opts), @@ -195,7 +210,6 @@ init([Host, Opts]) -> ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90), ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50), - ejabberd_hooks:add(reopen_log_hook, ?MODULE, reopen_log, 50), ejabberd_hooks:add(local_send_to_resource_hook, Host, mod_antispam_rtbl, @@ -203,17 +217,17 @@ init([Host, Opts]) -> 50), RTBLHost = gen_mod:get_opt(rtbl_host, Opts), RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), - InitState0 = + InitState = #state{host = Host, jid_set = JIDsSet, url_set = URLsSet, + dump_fd = mod_antispam_dump:init_dumping(Host), max_cache_size = gen_mod:get_opt(cache_size, Opts), blocked_domains = set_to_map(SpamDomainsSet), whitelist_domains = set_to_map(WhitelistDomains, false), rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode}, mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), - InitState = init_open_dump_file(DumpFile, InitState0), {ok, InitState} catch {Op, File, Reason} when Op == open; Op == read -> @@ -221,18 +235,6 @@ init([Host, Opts]) -> {stop, config_error} end. -init_open_dump_file(none, State) -> - State; -init_open_dump_file(DumpFile, State) -> - case filelib:ensure_dir(DumpFile) of - ok -> - ok; - {error, Reason} -> - Dirname = filename:dirname(DumpFile), - throw({open, Dirname, Reason}) - end, - open_dump_file(DumpFile, State). - -spec handle_call(term(), {pid(), term()}, state()) -> {reply, {spam_filter, term()}, state()} | {noreply, state()}. handle_call({check_jid, From}, _From, #state{jid_set = JIDsSet} = State) -> @@ -296,32 +298,21 @@ handle_call(Request, From, State) -> {noreply, State}. -spec handle_cast(term(), state()) -> {noreply, state()}. -handle_cast({dump, _XML}, #state{dump_fd = undefined} = State) -> - {noreply, State}; -handle_cast({dump, XML}, #state{dump_fd = Fd} = State) -> - case file:write(Fd, [XML, <<$\n>>]) of - ok -> - ok; - {error, Reason} -> - ?ERROR_MSG("Cannot write spam to dump file: ~s", [file:format_error(Reason)]) - end, +handle_cast({dump_stanza, XML}, #state{dump_fd = Fd} = State) -> + mod_antispam_dump:write_stanza_dump(Fd, XML), {noreply, State}; +handle_cast(reopen_log, #state{host = Host, dump_fd = Fd} = State) -> + {noreply, State#state{dump_fd = mod_antispam_dump:reopen_dump_file(Host, Fd)}}; handle_cast({reload, NewOpts, OldOpts}, #state{host = Host, + dump_fd = Fd, rtbl_host = OldRTBLHost, rtbl_domains_node = OldRTBLDomainsNode, rtbl_retry_timer = RTBLRetryTimer} = State) -> misc:cancel_timer(RTBLRetryTimer), State1 = - case {gen_mod:get_opt(spam_dump_file, OldOpts), gen_mod:get_opt(spam_dump_file, NewOpts)} - of - {OldDumpFile, NewDumpFile} when NewDumpFile /= OldDumpFile -> - close_dump_file(expand_host(OldDumpFile, Host), State), - open_dump_file(expand_host(NewDumpFile, Host), State); - {_OldDumpFile, _NewDumpFile} -> - State - end, + State#state{dump_fd = mod_antispam_dump:reload_dumping(Host, Fd, OldOpts, NewOpts)}, State2 = case {gen_mod:get_opt(cache_size, OldOpts), gen_mod:get_opt(cache_size, NewOpts)} of {OldMax, NewMax} when NewMax < OldMax -> @@ -342,8 +333,6 @@ handle_cast({reload, NewOpts, OldOpts}, RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, NewOpts), ok = mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), {noreply, State3#state{rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode}}; -handle_cast(reopen_log, State) -> - {noreply, reopen_dump_file(State)}; handle_cast({update_blocked_domains, NewItems}, #state{blocked_domains = BlockedDomains} = State) -> {noreply, State#state{blocked_domains = maps:merge(BlockedDomains, NewItems)}}; @@ -411,15 +400,14 @@ handle_info(Info, State) -> -spec terminate(normal | shutdown | {shutdown, term()} | term(), state()) -> ok. terminate(Reason, #state{host = Host, + dump_fd = Fd, rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode, rtbl_retry_timer = RTBLRetryTimer} = - State) -> + _State) -> ?DEBUG("Stopping spam filter process for ~s: ~p", [Host, Reason]), misc:cancel_timer(RTBLRetryTimer), - DumpFile = gen_mod:get_module_opt(Host, ?MODULE, spam_dump_file), - DumpFile1 = expand_host(DumpFile, Host), - close_dump_file(DumpFile1, State), + mod_antispam_dump:terminate_dumping(Host, Fd), ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50), ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90), @@ -429,12 +417,7 @@ terminate(Reason, pubsub_event_handler, 50), mod_antispam_rtbl:unsubscribe(RTBLHost, RTBLDomainsNode, Host), - case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of - false -> - ejabberd_hooks:delete(reopen_log_hook, ?MODULE, reopen_log, 50); - true -> - ok - end. + ok. -spec code_change({down, term()} | term(), state(), term()) -> {ok, state()}. code_change(_OldVsn, #state{host = Host} = State, _Extra) -> @@ -442,8 +425,7 @@ code_change(_OldVsn, #state{host = Host} = State, _Extra) -> {ok, State}. %%-------------------------------------------------------------------- -%% Hook callbacks. -%%-------------------------------------------------------------------- +%%| Hook callbacks -spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) -> {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}. @@ -505,17 +487,9 @@ s2s_in_handle_info(State, {_Ref, {spam_filter, _}}) -> s2s_in_handle_info(State, _) -> State. --spec reopen_log() -> ok. -reopen_log() -> - lists:foreach(fun(Host) -> - Proc = get_proc_name(Host), - gen_server:cast(Proc, reopen_log) - end, - get_spam_filter_hosts()). +%%-------------------------------------------------------------------- +%%| Internal functions -%%-------------------------------------------------------------------- -%% Internal functions. -%%-------------------------------------------------------------------- -spec needs_checking(jid(), jid()) -> boolean(). needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) -> case gen_mod:is_loaded(LServer, ?MODULE) of @@ -814,7 +788,7 @@ reject(#message{from = From, [jid:encode(From), jid:encode(To)]), Txt = <<"Your message is unsolicited">>, Err = xmpp:err_policy_violation(Txt, Lang), - maybe_dump_spam(Msg), + ejabberd_hooks:run(spam_stanza_rejected, To#jid.lserver, [Msg]), ejabberd_router:route_error(Msg, Err); reject(#presence{from = From, to = To, @@ -828,48 +802,6 @@ reject(#presence{from = From, reject(_) -> ok. --spec open_dump_file(filename(), state()) -> state(). -open_dump_file(none, State) -> - State#state{dump_fd = undefined}; -open_dump_file(Name, State) -> - Modes = [append, raw, binary, delayed_write], - case file:open(Name, Modes) of - {ok, Fd} -> - ?DEBUG("Opened ~s", [Name]), - State#state{dump_fd = Fd}; - {error, Reason} -> - ?ERROR_MSG("Cannot open dump file ~s: ~s", [Name, file:format_error(Reason)]), - State#state{dump_fd = undefined} - end. - --spec close_dump_file(filename(), state()) -> ok. -close_dump_file(_Name, #state{dump_fd = undefined}) -> - ok; -close_dump_file(Name, #state{dump_fd = Fd}) -> - case file:close(Fd) of - ok -> - ?DEBUG("Closed ~s", [Name]); - {error, Reason} -> - ?ERROR_MSG("Cannot close ~s: ~s", [Name, file:format_error(Reason)]) - end. - --spec reopen_dump_file(state()) -> state(). -reopen_dump_file(#state{host = Host} = State) -> - DumpFile = gen_mod:get_module_opt(Host, ?MODULE, spam_dump_file), - DumpFile1 = expand_host(DumpFile, Host), - close_dump_file(DumpFile1, State), - open_dump_file(DumpFile1, State). - --spec maybe_dump_spam(message()) -> ok. -maybe_dump_spam(#message{to = #jid{lserver = LServer}} = Msg) -> - By = jid:make(<<>>, LServer), - Proc = get_proc_name(LServer), - Time = erlang:timestamp(), - Msg1 = misc:add_delay_info(Msg, By, Time), - XML = fxml:element_to_binary( - xmpp:encode(Msg1)), - gen_server:cast(Proc, {dump, XML}). - -spec get_proc_name(binary()) -> atom(). get_proc_name(Host) -> gen_mod:get_module_proc(Host, ?MODULE). @@ -878,12 +810,6 @@ get_proc_name(Host) -> get_spam_filter_hosts() -> [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, ?MODULE)]. --spec expand_host(binary() | none, binary()) -> binary() | none. -expand_host(none, _Host) -> - none; -expand_host(Input, Host) -> - misc:expand_keyword(<<"@HOST@">>, Input, Host). - -spec sets_equal(sets:set(), sets:set()) -> boolean(). sets_equal(A, B) -> sets:is_subset(A, B) andalso sets:is_subset(B, A). @@ -901,8 +827,8 @@ format_error(Reason) -> list_to_binary(file:format_error(Reason)). %%-------------------------------------------------------------------- -%% Caching. -%%-------------------------------------------------------------------- +%%| Caching + -spec cache_insert(ljid(), state()) -> state(). cache_insert(_LJID, #state{max_cache_size = 0} = State) -> State; @@ -961,8 +887,8 @@ drop_from_cache(LJID, #state{jid_cache = Cache} = State) -> end. %%-------------------------------------------------------------------- -%% ejabberd command callbacks. -%%-------------------------------------------------------------------- +%%| ejabberd command callbacks + -spec get_commands_spec() -> [ejabberd_commands()]. get_commands_spec() -> [#ejabberd_commands{name = reload_spam_filter_files, @@ -1172,3 +1098,7 @@ drop_from_spam_filter_cache(Host, EncJID) -> _:{bad_jid, _} -> {error, "Not a valid JID: " ++ binary_to_list(EncJID)} end. + +%%-------------------------------------------------------------------- + +%%| vim: set foldmethod=marker foldmarker=%%|,%%-: diff --git a/src/mod_antispam_dump.erl b/src/mod_antispam_dump.erl new file mode 100644 index 000000000..134da7c74 --- /dev/null +++ b/src/mod_antispam_dump.erl @@ -0,0 +1,189 @@ +%%%---------------------------------------------------------------------- +%%% File : mod_antispam_dump.erl +%%% Author : Holger Weiss +%%% Author : Stefan Strigler +%%% Purpose : Filter spam messages based on sender JID and content +%%% Created : 31 Mar 2019 by Holger Weiss +%%% +%%% +%%% ejabberd, Copyright (C) 2019-2025 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%---------------------------------------------------------------------- + +%%| definitions +%% @format-begin + +-module(mod_antispam_dump). + +-author('holger@zedat.fu-berlin.de'). +-author('stefan@strigler.de'). + +-export([init_dumping/1, terminate_dumping/2, reload_dumping/4, reopen_dump_file/2, + write_stanza_dump/2]). +%% ejabberd_hooks callbacks. +-export([dump_spam_stanza/1, reopen_log/0]). + +-include("logger.hrl"). +-include("translate.hrl"). + +-include_lib("xmpp/include/xmpp.hrl"). + +-type filename() :: binary() | none | false. + +-define(MODULE_PARENT, mod_antispam). + +%%-------------------------------------------------------------------- +%%| Exported + +init_dumping(Host) -> + case get_path_option(Host) of + false -> + undefined; + DumpFile when is_binary(DumpFile) -> + case filelib:ensure_dir(DumpFile) of + ok -> + ejabberd_hooks:add(spam_stanza_rejected, Host, ?MODULE, dump_spam_stanza, 50), + ejabberd_hooks:add(reopen_log_hook, ?MODULE, reopen_log, 50), + open_dump_file(DumpFile); + {error, Reason} -> + Dirname = filename:dirname(DumpFile), + throw({open, Dirname, Reason}) + end + end. + +terminate_dumping(_Host, false) -> + ok; +terminate_dumping(Host, Fd) -> + DumpFile1 = get_path_option(Host), + close_dump_file(Fd, DumpFile1), + ejabberd_hooks:delete(spam_stanza_rejected, Host, ?MODULE, dump_spam_stanza, 50), + case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of + false -> + ejabberd_hooks:delete(reopen_log_hook, ?MODULE, reopen_log, 50); + true -> + ok + end. + +reload_dumping(Host, Fd, OldOpts, NewOpts) -> + case {get_path_option(Host, OldOpts), get_path_option(Host, NewOpts)} of + {Old, Old} -> + Fd; + {Old, New} -> + reopen_dump_file(Fd, Old, New) + end. + +-spec reopen_dump_file(binary(), file:io_device()) -> file:io_device(). +reopen_dump_file(Host, Fd) -> + DumpFile1 = get_path_option(Host), + reopen_dump_file(Fd, DumpFile1, DumpFile1). + +%%-------------------------------------------------------------------- +%%| Hook callbacks + +-spec dump_spam_stanza(message()) -> ok. +dump_spam_stanza(#message{to = #jid{lserver = LServer}} = Msg) -> + By = jid:make(<<>>, LServer), + Proc = get_proc_name(LServer), + Time = erlang:timestamp(), + Msg1 = misc:add_delay_info(Msg, By, Time), + XML = fxml:element_to_binary( + xmpp:encode(Msg1)), + gen_server:cast(Proc, {dump_stanza, XML}). + +-spec reopen_log() -> ok. +reopen_log() -> + lists:foreach(fun(Host) -> + Proc = get_proc_name(Host), + gen_server:cast(Proc, reopen_log) + end, + get_spam_filter_hosts()). + +%%-------------------------------------------------------------------- +%%| File management + +-spec open_dump_file(filename()) -> undefined | file:io_device(). +open_dump_file(false) -> + undefined; +open_dump_file(Name) -> + Modes = [append, raw, binary, delayed_write], + case file:open(Name, Modes) of + {ok, Fd} -> + ?DEBUG("Opened ~s", [Name]), + Fd; + {error, Reason} -> + ?ERROR_MSG("Cannot open dump file ~s: ~s", [Name, file:format_error(Reason)]), + undefined + end. + +-spec close_dump_file(undefined | file:io_device(), filename()) -> ok. +close_dump_file(undefined, false) -> + ok; +close_dump_file(Fd, Name) -> + case file:close(Fd) of + ok -> + ?DEBUG("Closed ~s", [Name]); + {error, Reason} -> + ?ERROR_MSG("Cannot close ~s: ~s", [Name, file:format_error(Reason)]) + end. + +-spec reopen_dump_file(file:io_device(), binary(), binary()) -> file:io_device(). +reopen_dump_file(Fd, OldDumpFile, NewDumpFile) -> + close_dump_file(Fd, OldDumpFile), + open_dump_file(NewDumpFile). + +write_stanza_dump(Fd, XML) -> + case file:write(Fd, [XML, <<$\n>>]) of + ok -> + ok; + {error, Reason} -> + ?ERROR_MSG("Cannot write spam to dump file: ~s", [file:format_error(Reason)]) + end. + +%%-------------------------------------------------------------------- +%%| Auxiliary + +get_path_option(Host) -> + Opts = gen_mod:get_module_opts(Host, ?MODULE_PARENT), + get_path_option(Host, Opts). + +get_path_option(Host, Opts) -> + case gen_mod:get_opt(spam_dump_file, Opts) of + false -> + false; + true -> + LogDirPath = + iolist_to_binary(filename:dirname( + ejabberd_logger:get_log_path())), + filename:join([LogDirPath, <<"spam_dump_", Host/binary, ".log">>]); + B when is_binary(B) -> + B + end. + +%%-------------------------------------------------------------------- +%%| Copied from mod_antispam.erl + +-spec get_proc_name(binary()) -> atom(). +get_proc_name(Host) -> + gen_mod:get_module_proc(Host, ?MODULE_PARENT). + +-spec get_spam_filter_hosts() -> [binary()]. +get_spam_filter_hosts() -> + [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, ?MODULE_PARENT)]. + +%%-------------------------------------------------------------------- + +%%| vim: set foldmethod=marker foldmarker=%%|,%%-: From d9a7b67f0e5755a19dc24d80d55fe74d373c63f3 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 17 Jun 2025 17:21:58 +0200 Subject: [PATCH 15/27] mod_antispam: increase timeout when waiting for dump file --- test/antispam_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index 0eb8d4041..280dacb97 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -247,7 +247,7 @@ rtbl_domains_whitelisted(Config) -> spam_dump_file(Config) -> {ok, CWD} = file:get_cwd(), Filename = filename:join([CWD, "spam.log"]), - ?retry(100, 10, + ?retry(100, 100, ?match(true, size(get_bytes(Filename)) > 0)), From = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>), To = my_jid(Config), From f3b1b5d41935ee4c6e85d4f1dab5ed8ddca9d3b5 Mon Sep 17 00:00:00 2001 From: Badlop Date: Wed, 18 Jun 2025 11:35:06 +0200 Subject: [PATCH 16/27] Result of running "make format" --- test/antispam_tests.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index 280dacb97..debfe9981 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -247,14 +247,13 @@ rtbl_domains_whitelisted(Config) -> spam_dump_file(Config) -> {ok, CWD} = file:get_cwd(), Filename = filename:join([CWD, "spam.log"]), - ?retry(100, 100, - ?match(true, size(get_bytes(Filename)) > 0)), + ?retry(100, 100, ?match(true, size(get_bytes(Filename)) > 0)), From = jid:make(<<"spammer_jid">>, <<"localhost">>, <<"spam_client">>), To = my_jid(Config), is_spam(message(From, To, <<"A very specific spam message">>)), - ?retry(100, 100, - ?match({match, _}, - re:run(get_bytes(Filename), <<"A very specific spam message">>))). + ?retry(100, + 100, + ?match({match, _}, re:run(get_bytes(Filename), <<"A very specific spam message">>))). %%%=================================================================== %%% Internal functions From 432810db894dcc0f1160eea34cbf4eec1f611e6f Mon Sep 17 00:00:00 2001 From: Badlop Date: Wed, 18 Jun 2025 11:34:34 +0200 Subject: [PATCH 17/27] Fix minor typos --- src/mod_antispam_dump.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mod_antispam_dump.erl b/src/mod_antispam_dump.erl index 134da7c74..04a671f05 100644 --- a/src/mod_antispam_dump.erl +++ b/src/mod_antispam_dump.erl @@ -2,7 +2,7 @@ %%% File : mod_antispam_dump.erl %%% Author : Holger Weiss %%% Author : Stefan Strigler -%%% Purpose : Filter spam messages based on sender JID and content +%%% Purpose : Manage dump file for filtered spam messages %%% Created : 31 Mar 2019 by Holger Weiss %%% %%% @@ -24,7 +24,7 @@ %%% %%%---------------------------------------------------------------------- -%%| definitions +%%| Definitions %% @format-begin -module(mod_antispam_dump). @@ -34,7 +34,7 @@ -export([init_dumping/1, terminate_dumping/2, reload_dumping/4, reopen_dump_file/2, write_stanza_dump/2]). -%% ejabberd_hooks callbacks. +%% ejabberd_hooks callbacks -export([dump_spam_stanza/1, reopen_log/0]). -include("logger.hrl"). From d00561b58c2d3f1ecf9d1ed1cda455f6ac08691c Mon Sep 17 00:00:00 2001 From: Badlop Date: Wed, 18 Jun 2025 10:32:49 +0200 Subject: [PATCH 18/27] Move filtering implementation to a submodule --- src/mod_antispam.erl | 253 ++---------------------------- src/mod_antispam_filter.erl | 299 ++++++++++++++++++++++++++++++++++++ test/antispam_tests.erl | 5 +- 3 files changed, 313 insertions(+), 244 deletions(-) create mode 100644 src/mod_antispam_filter.erl diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index 8b161b558..ef2106219 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -50,11 +50,6 @@ terminate/2, code_change/3]). -%% ejabberd_hooks callbacks. --export([s2s_in_handle_info/2, - s2s_receive_packet/1, - sm_receive_packet/1]). - %% ejabberd_commands callbacks. -export([add_blocked_domain/2, add_to_spam_filter_cache/2, @@ -76,7 +71,6 @@ -type filename() :: binary() | none | false. -type jid_set() :: sets:set(ljid()). -type url_set() :: sets:set(url()). --type s2s_in_state() :: ejabberd_s2s_in:state(). -record(state, {host = <<>> :: binary(), @@ -96,7 +90,6 @@ -type state() :: #state{}. -define(COMMAND_TIMEOUT, timer:seconds(30)). --define(HTTPC_TIMEOUT, timer:seconds(3)). -define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). -define(DEFAULT_CACHE_SIZE, 10000). @@ -178,7 +171,15 @@ mod_doc() -> #{desc => ?T("Reads from text file and RTBL, filters stanzas and writes dump file."), note => "added in 25.xx", opts => - [{spam_dump_file, + [{access_spam, + #{value => ?T("Access"), + desc => + ?T("Access rule that controls what accounts may receive spam messages. " + "If the rule returns `allow` for a given recipient, " + "spam messages aren't rejected for that recipient. " + "The default value is 'none', which means that all recipients " + "are subject to spam filtering verification.")}}, + {spam_dump_file, #{value => ?T("false | true | Path"), desc => ?T("Path to the file to store blocked messages. " @@ -207,9 +208,6 @@ init([Host, Opts]) -> url := URLsSet, domains := SpamDomainsSet, whitelist_domains := WhitelistDomains} -> - ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90), - ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), - ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50), ejabberd_hooks:add(local_send_to_resource_hook, Host, mod_antispam_rtbl, @@ -217,6 +215,7 @@ init([Host, Opts]) -> 50), RTBLHost = gen_mod:get_opt(rtbl_host, Opts), RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), + mod_antispam_filter:init_filtering(Host), InitState = #state{host = Host, jid_set = JIDsSet, @@ -252,9 +251,6 @@ handle_call({check_body, URLs, JIDs, From}, Result2 end, {reply, {spam_filter, Result}, State2}; -handle_call({resolve_redirects, URLs}, _From, State) -> - ResolvedURLs = do_resolve_redirects(URLs, []), - {reply, {spam_filter, ResolvedURLs}, State}; handle_call({reload_files, Files}, _From, State) -> {Result, State1} = reload_files(Files, State), {reply, {spam_filter, Result}, State1}; @@ -408,9 +404,7 @@ terminate(Reason, ?DEBUG("Stopping spam filter process for ~s: ~p", [Host, Reason]), misc:cancel_timer(RTBLRetryTimer), mod_antispam_dump:terminate_dumping(Host, Fd), - ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), - ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50), - ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90), + mod_antispam_filter:terminate_filtering(Host), ejabberd_hooks:delete(local_send_to_resource_hook, Host, mod_antispam_rtbl, @@ -424,209 +418,9 @@ code_change(_OldVsn, #state{host = Host} = State, _Extra) -> ?DEBUG("Updating spam filter process for ~s", [Host]), {ok, State}. -%%-------------------------------------------------------------------- -%%| Hook callbacks - --spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) -> - {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}. -s2s_receive_packet({A, State}) -> - case sm_receive_packet(A) of - {stop, drop} -> - {stop, {drop, State}}; - Result -> - {Result, State} - end. - --spec sm_receive_packet(stanza() | drop) -> stanza() | drop | {stop, drop}. -sm_receive_packet(drop = Acc) -> - Acc; -sm_receive_packet(#message{from = From, - to = #jid{lserver = LServer} = To, - type = Type} = - Msg) - when Type /= groupchat, Type /= error -> - do_check(From, To, LServer, Msg); -sm_receive_packet(#presence{from = From, - to = #jid{lserver = LServer} = To, - type = subscribe} = - Presence) -> - do_check(From, To, LServer, Presence); -sm_receive_packet(Acc) -> - Acc. - -do_check(From, To, LServer, Stanza) -> - case needs_checking(From, To) of - true -> - case check_from(LServer, From) of - ham -> - case check_stanza(LServer, From, Stanza) of - ham -> - Stanza; - spam -> - reject(Stanza), - {stop, drop} - end; - spam -> - reject(Stanza), - {stop, drop} - end; - false -> - Stanza - end. - -check_stanza(LServer, From, #message{body = Body}) -> - check_body(LServer, From, xmpp:get_text(Body)); -check_stanza(_, _, _) -> - ham. - --spec s2s_in_handle_info(s2s_in_state(), any()) -> - s2s_in_state() | {stop, s2s_in_state()}. -s2s_in_handle_info(State, {_Ref, {spam_filter, _}}) -> - ?DEBUG("Dropping expired spam filter result", []), - {stop, State}; -s2s_in_handle_info(State, _) -> - State. - %%-------------------------------------------------------------------- %%| Internal functions --spec needs_checking(jid(), jid()) -> boolean(). -needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) -> - case gen_mod:is_loaded(LServer, ?MODULE) of - true -> - Access = gen_mod:get_module_opt(LServer, ?MODULE, access_spam), - case acl:match_rule(LServer, Access, To) of - allow -> - ?DEBUG("Spam not filtered for ~s", [jid:encode(To)]), - false; - deny -> - ?DEBUG("Spam is filtered for ~s", [jid:encode(To)]), - not mod_roster:is_subscribed(From, To) - andalso not - mod_roster:is_subscribed( - jid:make(<<>>, FromHost), - To) % likely a gateway - end; - false -> - ?DEBUG("~s not loaded for ~s", [?MODULE, LServer]), - false - end. - --spec check_from(binary(), jid()) -> ham | spam. -check_from(Host, From) -> - Proc = get_proc_name(Host), - LFrom = - {_, FromDomain, _} = - jid:remove_resource( - jid:tolower(From)), - try - case gen_server:call(Proc, {is_blocked_domain, FromDomain}) of - true -> - ?DEBUG("Spam JID found in blocked domains: ~p", [From]), - ejabberd_hooks:run(spam_found, Host, [{jid, From}]), - spam; - false -> - case gen_server:call(Proc, {check_jid, LFrom}) of - {spam_filter, Result} -> - Result - end - end - catch - exit:{timeout, _} -> - ?WARNING_MSG("Timeout while checking ~s against list of blocked domains or spammers", - [jid:encode(From)]), - ham - end. - --spec check_body(binary(), jid(), binary()) -> ham | spam. -check_body(Host, From, Body) -> - case {extract_urls(Host, Body), extract_jids(Body)} of - {none, none} -> - ?DEBUG("No JIDs/URLs found in message", []), - ham; - {URLs, JIDs} -> - Proc = get_proc_name(Host), - LFrom = - jid:remove_resource( - jid:tolower(From)), - try gen_server:call(Proc, {check_body, URLs, JIDs, LFrom}) of - {spam_filter, Result} -> - Result - catch - exit:{timeout, _} -> - ?WARNING_MSG("Timeout while checking body", []), - ham - end - end. - --spec extract_urls(binary(), binary()) -> {urls, [url()]} | none. -extract_urls(Host, Body) -> - RE = <<"https?://\\S+">>, - Options = [global, {capture, all, binary}], - case re:run(Body, RE, Options) of - {match, Captured} when is_list(Captured) -> - Urls = resolve_redirects(Host, lists:flatten(Captured)), - {urls, Urls}; - nomatch -> - none - end. - --spec resolve_redirects(binary(), [url()]) -> [url()]. -resolve_redirects(Host, URLs) -> - Proc = get_proc_name(Host), - try gen_server:call(Proc, {resolve_redirects, URLs}) of - {spam_filter, ResolvedURLs} -> - ResolvedURLs - catch - exit:{timeout, _} -> - ?WARNING_MSG("Timeout while resolving redirects: ~p", [URLs]), - URLs - end. - --spec do_resolve_redirects([url()], [url()]) -> [url()]. -do_resolve_redirects([], Result) -> - Result; -do_resolve_redirects([URL | Rest], Acc) -> - case httpc:request(get, - {URL, [{"user-agent", "curl/8.7.1"}]}, - [{autoredirect, false}, {timeout, ?HTTPC_TIMEOUT}], - []) - of - {ok, {{_, StatusCode, _}, Headers, _Body}} when StatusCode >= 300, StatusCode < 400 -> - Location = proplists:get_value("location", Headers), - case Location == undefined orelse lists:member(Location, Acc) of - true -> - do_resolve_redirects(Rest, [URL | Acc]); - false -> - do_resolve_redirects([Location | Rest], [URL | Acc]) - end; - _Res -> - do_resolve_redirects(Rest, [URL | Acc]) - end. - --spec extract_jids(binary()) -> {jids, [ljid()]} | none. -extract_jids(Body) -> - RE = <<"\\S+@\\S+">>, - Options = [global, {capture, all, binary}], - case re:run(Body, RE, Options) of - {match, Captured} when is_list(Captured) -> - {jids, lists:filtermap(fun try_decode_jid/1, lists:flatten(Captured))}; - nomatch -> - none - end. - --spec try_decode_jid(binary()) -> {true, ljid()} | false. -try_decode_jid(S) -> - try jid:decode(S) of - #jid{} = JID -> - {true, - jid:remove_resource( - jid:tolower(JID))} - catch - _:{bad_jid, _} -> - false - end. - -spec filter_jid(ljid(), jid_set(), state()) -> {ham | spam, state()}. filter_jid(From, Set, #state{host = Host} = State) -> case sets:is_element(From, Set) of @@ -777,31 +571,6 @@ parse_url(S) -> trim(S) -> re:replace(S, <<"\\s+$">>, <<>>, [{return, binary}]). --spec reject(stanza()) -> ok. -reject(#message{from = From, - to = To, - type = Type, - lang = Lang} = - Msg) - when Type /= groupchat, Type /= error -> - ?INFO_MSG("Rejecting unsolicited message from ~s to ~s", - [jid:encode(From), jid:encode(To)]), - Txt = <<"Your message is unsolicited">>, - Err = xmpp:err_policy_violation(Txt, Lang), - ejabberd_hooks:run(spam_stanza_rejected, To#jid.lserver, [Msg]), - ejabberd_router:route_error(Msg, Err); -reject(#presence{from = From, - to = To, - lang = Lang} = - Presence) -> - ?INFO_MSG("Rejecting unsolicited presence from ~s to ~s", - [jid:encode(From), jid:encode(To)]), - Txt = <<"Your traffic is unsolicited">>, - Err = xmpp:err_policy_violation(Txt, Lang), - ejabberd_router:route_error(Presence, Err); -reject(_) -> - ok. - -spec get_proc_name(binary()) -> atom(). get_proc_name(Host) -> gen_mod:get_module_proc(Host, ?MODULE). diff --git a/src/mod_antispam_filter.erl b/src/mod_antispam_filter.erl new file mode 100644 index 000000000..9fd8abf36 --- /dev/null +++ b/src/mod_antispam_filter.erl @@ -0,0 +1,299 @@ +%%%---------------------------------------------------------------------- +%%% File : mod_antispam_filter.erl +%%% Author : Holger Weiss +%%% Author : Stefan Strigler +%%% Purpose : Filter C2S and S2S stanzas +%%% Created : 31 Mar 2019 by Holger Weiss +%%% +%%% +%%% ejabberd, Copyright (C) 2019-2025 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%---------------------------------------------------------------------- + +%%| Definitions +%% @format-begin + +-module(mod_antispam_filter). + +-author('holger@zedat.fu-berlin.de'). +-author('stefan@strigler.de'). + +-export([init_filtering/1, terminate_filtering/1]). +%% ejabberd_hooks callbacks +-export([s2s_in_handle_info/2, s2s_receive_packet/1, sm_receive_packet/1]). + +-include("logger.hrl"). +-include("translate.hrl"). + +-include_lib("xmpp/include/xmpp.hrl"). + +-type url() :: binary(). +-type s2s_in_state() :: ejabberd_s2s_in:state(). + +-define(MODULE_PARENT, mod_antispam). +-define(HTTPC_TIMEOUT, timer:seconds(3)). + +%%-------------------------------------------------------------------- +%%| Exported + +init_filtering(Host) -> + ejabberd_hooks:add(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90), + ejabberd_hooks:add(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), + ejabberd_hooks:add(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50). + +terminate_filtering(Host) -> + ejabberd_hooks:delete(s2s_receive_packet, Host, ?MODULE, s2s_receive_packet, 50), + ejabberd_hooks:delete(sm_receive_packet, Host, ?MODULE, sm_receive_packet, 50), + ejabberd_hooks:delete(s2s_in_handle_info, Host, ?MODULE, s2s_in_handle_info, 90). + +%%-------------------------------------------------------------------- +%%| Hook callbacks + +-spec s2s_receive_packet({stanza() | drop, s2s_in_state()}) -> + {stanza() | drop, s2s_in_state()} | {stop, {drop, s2s_in_state()}}. +s2s_receive_packet({A, State}) -> + case sm_receive_packet(A) of + {stop, drop} -> + {stop, {drop, State}}; + Result -> + {Result, State} + end. + +-spec sm_receive_packet(stanza() | drop) -> stanza() | drop | {stop, drop}. +sm_receive_packet(drop = Acc) -> + Acc; +sm_receive_packet(#message{from = From, + to = #jid{lserver = LServer} = To, + type = Type} = + Msg) + when Type /= groupchat, Type /= error -> + do_check(From, To, LServer, Msg); +sm_receive_packet(#presence{from = From, + to = #jid{lserver = LServer} = To, + type = subscribe} = + Presence) -> + do_check(From, To, LServer, Presence); +sm_receive_packet(Acc) -> + Acc. + +%%-------------------------------------------------------------------- +%%| Filtering deciding + +do_check(From, To, LServer, Stanza) -> + case needs_checking(From, To) of + true -> + case check_from(LServer, From) of + ham -> + case check_stanza(LServer, From, Stanza) of + ham -> + Stanza; + spam -> + reject(Stanza), + {stop, drop} + end; + spam -> + reject(Stanza), + {stop, drop} + end; + false -> + Stanza + end. + +check_stanza(LServer, From, #message{body = Body}) -> + check_body(LServer, From, xmpp:get_text(Body)); +check_stanza(_, _, _) -> + ham. + +-spec s2s_in_handle_info(s2s_in_state(), any()) -> + s2s_in_state() | {stop, s2s_in_state()}. +s2s_in_handle_info(State, {_Ref, {spam_filter, _}}) -> + ?DEBUG("Dropping expired spam filter result", []), + {stop, State}; +s2s_in_handle_info(State, _) -> + State. + +-spec needs_checking(jid(), jid()) -> boolean(). +needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) -> + case gen_mod:is_loaded(LServer, ?MODULE_PARENT) of + true -> + Access = gen_mod:get_module_opt(LServer, ?MODULE_PARENT, access_spam), + case acl:match_rule(LServer, Access, To) of + allow -> + ?DEBUG("Spam not filtered for ~s", [jid:encode(To)]), + false; + deny -> + ?DEBUG("Spam is filtered for ~s", [jid:encode(To)]), + not mod_roster:is_subscribed(From, To) + andalso not + mod_roster:is_subscribed( + jid:make(<<>>, FromHost), + To) % likely a gateway + end; + false -> + ?DEBUG("~s not loaded for ~s", [?MODULE_PARENT, LServer]), + false + end. + +-spec check_from(binary(), jid()) -> ham | spam. +check_from(Host, From) -> + Proc = get_proc_name(Host), + LFrom = + {_, FromDomain, _} = + jid:remove_resource( + jid:tolower(From)), + try + case gen_server:call(Proc, {is_blocked_domain, FromDomain}) of + true -> + ?DEBUG("Spam JID found in blocked domains: ~p", [From]), + ejabberd_hooks:run(spam_found, Host, [{jid, From}]), + spam; + false -> + case gen_server:call(Proc, {check_jid, LFrom}) of + {spam_filter, Result} -> + Result + end + end + catch + exit:{timeout, _} -> + ?WARNING_MSG("Timeout while checking ~s against list of blocked domains or spammers", + [jid:encode(From)]), + ham + end. + +-spec check_body(binary(), jid(), binary()) -> ham | spam. +check_body(Host, From, Body) -> + case {extract_urls(Host, Body), extract_jids(Body)} of + {none, none} -> + ?DEBUG("No JIDs/URLs found in message", []), + ham; + {URLs, JIDs} -> + Proc = get_proc_name(Host), + LFrom = + jid:remove_resource( + jid:tolower(From)), + try gen_server:call(Proc, {check_body, URLs, JIDs, LFrom}) of + {spam_filter, Result} -> + Result + catch + exit:{timeout, _} -> + ?WARNING_MSG("Timeout while checking body", []), + ham + end + end. + +%%-------------------------------------------------------------------- +%%| Auxiliary + +-spec extract_urls(binary(), binary()) -> {urls, [url()]} | none. +extract_urls(Host, Body) -> + RE = <<"https?://\\S+">>, + Options = [global, {capture, all, binary}], + case re:run(Body, RE, Options) of + {match, Captured} when is_list(Captured) -> + Urls = resolve_redirects(Host, lists:flatten(Captured)), + {urls, Urls}; + nomatch -> + none + end. + +-spec resolve_redirects(binary(), [url()]) -> [url()]. +resolve_redirects(_Host, URLs) -> + try do_resolve_redirects(URLs, []) of + ResolvedURLs -> + ResolvedURLs + catch + exit:{timeout, _} -> + ?WARNING_MSG("Timeout while resolving redirects: ~p", [URLs]), + URLs + end. + +-spec do_resolve_redirects([url()], [url()]) -> [url()]. +do_resolve_redirects([], Result) -> + Result; +do_resolve_redirects([URL | Rest], Acc) -> + case httpc:request(get, + {URL, [{"user-agent", "curl/8.7.1"}]}, + [{autoredirect, false}, {timeout, ?HTTPC_TIMEOUT}], + []) + of + {ok, {{_, StatusCode, _}, Headers, _Body}} when StatusCode >= 300, StatusCode < 400 -> + Location = proplists:get_value("location", Headers), + case Location == undefined orelse lists:member(Location, Acc) of + true -> + do_resolve_redirects(Rest, [URL | Acc]); + false -> + do_resolve_redirects([Location | Rest], [URL | Acc]) + end; + _Res -> + do_resolve_redirects(Rest, [URL | Acc]) + end. + +-spec extract_jids(binary()) -> {jids, [ljid()]} | none. +extract_jids(Body) -> + RE = <<"\\S+@\\S+">>, + Options = [global, {capture, all, binary}], + case re:run(Body, RE, Options) of + {match, Captured} when is_list(Captured) -> + {jids, lists:filtermap(fun try_decode_jid/1, lists:flatten(Captured))}; + nomatch -> + none + end. + +-spec try_decode_jid(binary()) -> {true, ljid()} | false. +try_decode_jid(S) -> + try jid:decode(S) of + #jid{} = JID -> + {true, + jid:remove_resource( + jid:tolower(JID))} + catch + _:{bad_jid, _} -> + false + end. + +-spec reject(stanza()) -> ok. +reject(#message{from = From, + to = To, + type = Type, + lang = Lang} = + Msg) + when Type /= groupchat, Type /= error -> + ?INFO_MSG("Rejecting unsolicited message from ~s to ~s", + [jid:encode(From), jid:encode(To)]), + Txt = <<"Your message is unsolicited">>, + Err = xmpp:err_policy_violation(Txt, Lang), + ejabberd_hooks:run(spam_stanza_rejected, To#jid.lserver, [Msg]), + ejabberd_router:route_error(Msg, Err); +reject(#presence{from = From, + to = To, + lang = Lang} = + Presence) -> + ?INFO_MSG("Rejecting unsolicited presence from ~s to ~s", + [jid:encode(From), jid:encode(To)]), + Txt = <<"Your traffic is unsolicited">>, + Err = xmpp:err_policy_violation(Txt, Lang), + ejabberd_router:route_error(Presence, Err); +reject(_) -> + ok. + +-spec get_proc_name(binary()) -> atom(). +get_proc_name(Host) -> + gen_mod:get_module_proc(Host, ?MODULE_PARENT). + +%%-------------------------------------------------------------------- + +%%| vim: set foldmethod=marker foldmarker=%%|,%%-: diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index debfe9981..f60872913 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -265,10 +265,11 @@ has_spam_domain(Domain) -> fun(Host) -> lists:member(Domain, mod_antispam:get_blocked_domains(Host)) end. is_not_spam(Msg) -> - ?match({Msg, undefined}, mod_antispam:s2s_receive_packet({Msg, undefined})). + ?match({Msg, undefined}, mod_antispam_filter:s2s_receive_packet({Msg, undefined})). is_spam(Spam) -> - ?match({stop, {drop, undefined}}, mod_antispam:s2s_receive_packet({Spam, undefined})). + ?match({stop, {drop, undefined}}, + mod_antispam_filter:s2s_receive_packet({Spam, undefined})). message_hello(Username, Host, Config) -> SpamFrom = jid:make(Username, Host, <<"spam_client">>), From bddcf0624ee856efe1679c1ac154a539e4b8a3c2 Mon Sep 17 00:00:00 2001 From: Badlop Date: Sat, 21 Jun 2025 18:27:38 +0200 Subject: [PATCH 19/27] mod_antispam: Move some definitions to a header file --- include/mod_antispam.hrl | 26 ++++++++++++++++++++++++++ src/mod_antispam.erl | 6 +----- src/mod_antispam_dump.erl | 11 ++++------- src/mod_antispam_filter.erl | 11 +++++------ src/mod_antispam_rtbl.erl | 1 + 5 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 include/mod_antispam.hrl diff --git a/include/mod_antispam.hrl b/include/mod_antispam.hrl new file mode 100644 index 000000000..7f681046e --- /dev/null +++ b/include/mod_antispam.hrl @@ -0,0 +1,26 @@ +%%%---------------------------------------------------------------------- +%%% +%%% ejabberd, Copyright (C) 2002-2025 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%---------------------------------------------------------------------- + +-define(MODULE_ANTISPAM, mod_antispam). + +-type url() :: binary(). +-type filename() :: binary() | none | false. +-type jid_set() :: sets:set(ljid()). +-type url_set() :: sets:set(url()). diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index ef2106219..a48f7235c 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -63,15 +63,11 @@ -include("ejabberd_commands.hrl"). -include("logger.hrl"). +-include("mod_antispam.hrl"). -include("translate.hrl"). -include_lib("xmpp/include/xmpp.hrl"). --type url() :: binary(). --type filename() :: binary() | none | false. --type jid_set() :: sets:set(ljid()). --type url_set() :: sets:set(url()). - -record(state, {host = <<>> :: binary(), dump_fd = undefined :: file:io_device() | undefined, diff --git a/src/mod_antispam_dump.erl b/src/mod_antispam_dump.erl index 04a671f05..f92ac0bf1 100644 --- a/src/mod_antispam_dump.erl +++ b/src/mod_antispam_dump.erl @@ -38,14 +38,11 @@ -export([dump_spam_stanza/1, reopen_log/0]). -include("logger.hrl"). +-include("mod_antispam.hrl"). -include("translate.hrl"). -include_lib("xmpp/include/xmpp.hrl"). --type filename() :: binary() | none | false. - --define(MODULE_PARENT, mod_antispam). - %%-------------------------------------------------------------------- %%| Exported @@ -157,7 +154,7 @@ write_stanza_dump(Fd, XML) -> %%| Auxiliary get_path_option(Host) -> - Opts = gen_mod:get_module_opts(Host, ?MODULE_PARENT), + Opts = gen_mod:get_module_opts(Host, ?MODULE_ANTISPAM), get_path_option(Host, Opts). get_path_option(Host, Opts) -> @@ -178,11 +175,11 @@ get_path_option(Host, Opts) -> -spec get_proc_name(binary()) -> atom(). get_proc_name(Host) -> - gen_mod:get_module_proc(Host, ?MODULE_PARENT). + gen_mod:get_module_proc(Host, ?MODULE_ANTISPAM). -spec get_spam_filter_hosts() -> [binary()]. get_spam_filter_hosts() -> - [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, ?MODULE_PARENT)]. + [H || H <- ejabberd_option:hosts(), gen_mod:is_loaded(H, ?MODULE_ANTISPAM)]. %%-------------------------------------------------------------------- diff --git a/src/mod_antispam_filter.erl b/src/mod_antispam_filter.erl index 9fd8abf36..7367c2ba8 100644 --- a/src/mod_antispam_filter.erl +++ b/src/mod_antispam_filter.erl @@ -38,13 +38,12 @@ -include("logger.hrl"). -include("translate.hrl"). +-include("mod_antispam.hrl"). -include_lib("xmpp/include/xmpp.hrl"). --type url() :: binary(). -type s2s_in_state() :: ejabberd_s2s_in:state(). --define(MODULE_PARENT, mod_antispam). -define(HTTPC_TIMEOUT, timer:seconds(3)). %%-------------------------------------------------------------------- @@ -128,9 +127,9 @@ s2s_in_handle_info(State, _) -> -spec needs_checking(jid(), jid()) -> boolean(). needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) -> - case gen_mod:is_loaded(LServer, ?MODULE_PARENT) of + case gen_mod:is_loaded(LServer, ?MODULE_ANTISPAM) of true -> - Access = gen_mod:get_module_opt(LServer, ?MODULE_PARENT, access_spam), + Access = gen_mod:get_module_opt(LServer, ?MODULE_ANTISPAM, access_spam), case acl:match_rule(LServer, Access, To) of allow -> ?DEBUG("Spam not filtered for ~s", [jid:encode(To)]), @@ -144,7 +143,7 @@ needs_checking(#jid{lserver = FromHost} = From, #jid{lserver = LServer} = To) -> To) % likely a gateway end; false -> - ?DEBUG("~s not loaded for ~s", [?MODULE_PARENT, LServer]), + ?DEBUG("~s not loaded for ~s", [?MODULE_ANTISPAM, LServer]), false end. @@ -292,7 +291,7 @@ reject(_) -> -spec get_proc_name(binary()) -> atom(). get_proc_name(Host) -> - gen_mod:get_module_proc(Host, ?MODULE_PARENT). + gen_mod:get_module_proc(Host, ?MODULE_ANTISPAM). %%-------------------------------------------------------------------- diff --git a/src/mod_antispam_rtbl.erl b/src/mod_antispam_rtbl.erl index 93b346631..246b59dfa 100644 --- a/src/mod_antispam_rtbl.erl +++ b/src/mod_antispam_rtbl.erl @@ -27,6 +27,7 @@ -include_lib("xmpp/include/xmpp.hrl"). -include("logger.hrl"). +-include("mod_antispam.hrl"). -define(SERVICE_MODULE, mod_antispam). -define(SERVICE_JID_PREFIX, "rtbl-"). From 88ae3fddf3bceb86cd8d3e0ac2e5f5a7bb756b7d Mon Sep 17 00:00:00 2001 From: Badlop Date: Sat, 21 Jun 2025 23:33:09 +0200 Subject: [PATCH 20/27] mod_antispam: Sort and document files options --- src/mod_antispam.erl | 80 +++++++++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index a48f7235c..fda470953 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -125,10 +125,17 @@ depends(_Host, _Opts) -> [{mod_pubsub, soft}]. -spec mod_opt_type(atom()) -> econf:validator(). -mod_opt_type(spam_domains_file) -> +mod_opt_type(access_spam) -> + econf:acl(); +mod_opt_type(cache_size) -> + econf:pos_int(unlimited); +mod_opt_type(rtbl_host) -> econf:either( - econf:enum([none]), econf:file()); -mod_opt_type(whitelist_domains_file) -> + econf:enum([none]), econf:host()); +mod_opt_type(rtbl_domains_node) -> + econf:non_empty( + econf:binary()); +mod_opt_type(spam_domains_file) -> econf:either( econf:enum([none]), econf:file()); mod_opt_type(spam_dump_file) -> @@ -140,28 +147,21 @@ mod_opt_type(spam_jids_file) -> mod_opt_type(spam_urls_file) -> econf:either( econf:enum([none]), econf:file()); -mod_opt_type(access_spam) -> - econf:acl(); -mod_opt_type(cache_size) -> - econf:pos_int(unlimited); -mod_opt_type(rtbl_host) -> +mod_opt_type(whitelist_domains_file) -> econf:either( - econf:enum([none]), econf:host()); -mod_opt_type(rtbl_domains_node) -> - econf:non_empty( - econf:binary()). + econf:enum([none]), econf:file()). -spec mod_options(binary()) -> [{atom(), any()}]. mod_options(_Host) -> - [{spam_domains_file, none}, + [{access_spam, none}, + {cache_size, ?DEFAULT_CACHE_SIZE}, + {rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}, + {rtbl_host, none}, + {spam_domains_file, none}, {spam_dump_file, false}, {spam_jids_file, none}, {spam_urls_file, none}, - {whitelist_domains_file, none}, - {access_spam, none}, - {cache_size, ?DEFAULT_CACHE_SIZE}, - {rtbl_host, none}, - {rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}]. + {whitelist_domains_file, none}]. mod_doc() -> #{desc => ?T("Reads from text file and RTBL, filters stanzas and writes dump file."), @@ -171,10 +171,20 @@ mod_doc() -> #{value => ?T("Access"), desc => ?T("Access rule that controls what accounts may receive spam messages. " - "If the rule returns `allow` for a given recipient, " + "If the rule returns 'allow' for a given recipient, " "spam messages aren't rejected for that recipient. " "The default value is 'none', which means that all recipients " "are subject to spam filtering verification.")}}, + {spam_domains_file, + #{value => ?T("none | Path"), + desc => + ?T("Path to a plain text file containing a list of " + "known spam domains, one domain per line. " + "Messages and subscription requests sent from one of the listed domains " + "are classified as spam if sender is not in recipient's roster. " + "This list of domains gets merged with the one retrieved " + "by an RTBL host if any given. " + "The default value is 'none'.")}}, {spam_dump_file, #{value => ?T("false | true | Path"), desc => @@ -182,7 +192,37 @@ mod_doc() -> "Use an absolute path, or the '@LOG_PATH@' macro to store logs " "in the same place that the other ejabberd log files. " "If set to 'false', does not dump stanzas, this is the default. " - "If set to 'true', it stores in '\"@LOG_PATH@/spam_dump_@HOST@.log\"'.")}}], + "If set to 'true', it stores in '\"@LOG_PATH@/spam_dump_@HOST@.log\"'.")}}, + {spam_jids_file, + #{value => ?T("none | Path"), + desc => + ?T("Path to a plain text file containing a list of " + "known spammer JIDs, one JID per line. " + "Messages and subscription requests sent from one of " + "the listed JIDs are classified as spam. " + "Messages containing at least one of the listed JIDs" + "are classified as spam as well. " + "Furthermore, the sender's JID will be cached, " + "so that future traffic originating from that JID will also be classified as spam. " + "The default value is 'none'.")}}, + {spam_urls_file, + #{value => ?T("none | Path"), + desc => + ?T("Path to a plain text file containing a list of " + "URLs known to be mentioned in spam message bodies. " + "Messages containing at least one of the listed URLs are classified as spam. " + "Furthermore, the sender's JID will be cached, " + "so that future traffic originating from that JID will be classified as spam as well. " + "The default value is 'none'.")}}, + {whitelist_domains_file, + #{value => ?T("none | Path"), + desc => + ?T("Path to a file containing a list of " + "domains to whitelist from being blocked, one per line. " + "If either it is in 'spam_domains_file' or more realistically " + "in a domain sent by a RTBL host (see option 'rtbl_host') " + "then this domain will be ignored and stanzas from there won't be blocked. " + "The default value is 'none'.")}}], example => ["modules:", " mod_antispam:", From a77c7e36b005fa9492102cdd4c94bc47f02bfd7e Mon Sep 17 00:00:00 2001 From: Badlop Date: Sat, 21 Jun 2025 23:33:35 +0200 Subject: [PATCH 21/27] Move spam files parsing to a submodule --- src/mod_antispam.erl | 193 ++++++++++--------------------------- src/mod_antispam_files.erl | 182 ++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 142 deletions(-) create mode 100644 src/mod_antispam_files.erl diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index fda470953..500cb0244 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -234,41 +234,33 @@ mod_doc() -> -spec init(list()) -> {ok, state()} | {stop, term()}. init([Host, Opts]) -> process_flag(trap_exit, true), - Files = - #{domains => gen_mod:get_opt(spam_domains_file, Opts), - jid => gen_mod:get_opt(spam_jids_file, Opts), - url => gen_mod:get_opt(spam_urls_file, Opts), - whitelist_domains => gen_mod:get_opt(whitelist_domains_file, Opts)}, - try read_files(Files) of - #{jid := JIDsSet, - url := URLsSet, - domains := SpamDomainsSet, - whitelist_domains := WhitelistDomains} -> - ejabberd_hooks:add(local_send_to_resource_hook, - Host, - mod_antispam_rtbl, - pubsub_event_handler, - 50), - RTBLHost = gen_mod:get_opt(rtbl_host, Opts), - RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), - mod_antispam_filter:init_filtering(Host), - InitState = - #state{host = Host, - jid_set = JIDsSet, - url_set = URLsSet, - dump_fd = mod_antispam_dump:init_dumping(Host), - max_cache_size = gen_mod:get_opt(cache_size, Opts), - blocked_domains = set_to_map(SpamDomainsSet), - whitelist_domains = set_to_map(WhitelistDomains, false), - rtbl_host = RTBLHost, - rtbl_domains_node = RTBLDomainsNode}, - mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), - {ok, InitState} - catch - {Op, File, Reason} when Op == open; Op == read -> - ?CRITICAL_MSG("Cannot ~s ~s: ~s", [Op, File, format_error(Reason)]), - {stop, config_error} - end. + mod_antispam_files:init_files(Host), + FilesResults = read_files(Host), + #{jid := JIDsSet, + url := URLsSet, + domains := SpamDomainsSet, + whitelist_domains := WhitelistDomains} = + FilesResults, + ejabberd_hooks:add(local_send_to_resource_hook, + Host, + mod_antispam_rtbl, + pubsub_event_handler, + 50), + RTBLHost = gen_mod:get_opt(rtbl_host, Opts), + RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), + mod_antispam_filter:init_filtering(Host), + InitState = + #state{host = Host, + jid_set = JIDsSet, + url_set = URLsSet, + dump_fd = mod_antispam_dump:init_dumping(Host), + max_cache_size = gen_mod:get_opt(cache_size, Opts), + blocked_domains = set_to_map(SpamDomainsSet), + whitelist_domains = set_to_map(WhitelistDomains, false), + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode}, + mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), + {ok, InitState}. -spec handle_call(term(), {pid(), term()}, state()) -> {reply, {spam_filter, term()}, state()} | {noreply, state()}. @@ -287,8 +279,8 @@ handle_call({check_body, URLs, JIDs, From}, Result2 end, {reply, {spam_filter, Result}, State2}; -handle_call({reload_files, Files}, _From, State) -> - {Result, State1} = reload_files(Files, State), +handle_call(reload_spam_files, _From, State) -> + {Result, State1} = reload_files(State), {reply, {spam_filter, Result}, State1}; handle_call({expire_cache, Age}, _From, State) -> {Result, State1} = expire_cache(Age, State), @@ -355,12 +347,7 @@ handle_cast({reload, NewOpts, OldOpts}, State1 end, ok = mod_antispam_rtbl:unsubscribe(OldRTBLHost, OldRTBLDomainsNode, Host), - Files = - #{domains => gen_mod:get_opt(spam_domains_file, NewOpts), - jid => gen_mod:get_opt(spam_jids_file, NewOpts), - url => gen_mod:get_opt(spam_urls_file, NewOpts), - whitelist_domains => gen_mod:get_opt(whitelist_domains_file, NewOpts)}, - {_Result, State3} = reload_files(Files, State2#state{blocked_domains = #{}}), + {_Result, State3} = reload_files(State2#state{blocked_domains = #{}}), RTBLHost = gen_mod:get_opt(rtbl_host, NewOpts), RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, NewOpts), ok = mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), @@ -440,6 +427,7 @@ terminate(Reason, ?DEBUG("Stopping spam filter process for ~s: ~p", [Host, Reason]), misc:cancel_timer(RTBLRetryTimer), mod_antispam_dump:terminate_dumping(Host, Fd), + mod_antispam_files:terminate_files(Host), mod_antispam_filter:terminate_filtering(Host), ejabberd_hooks:delete(local_send_to_resource_hook, Host, @@ -494,10 +482,9 @@ filter_body({_, Addrs}, Set, From, #state{host = Host} = State) -> filter_body(none, _Set, _From, State) -> {ham, State}. --spec reload_files(#{Type :: atom() => filename()}, state()) -> - {ok | {error, binary()}, state()}. -reload_files(Files, #state{host = Host, blocked_domains = BlockedDomains} = State) -> - try read_files(Files) of +-spec reload_files(state()) -> {ok | {error, binary()}, state()}. +reload_files(#state{host = Host, blocked_domains = BlockedDomains} = State) -> + case read_files(Host) of #{jid := JIDsSet, url := URLsSet, domains := SpamDomainsSet, @@ -518,12 +505,9 @@ reload_files(Files, #state{host = Host, blocked_domains = BlockedDomains} = Stat State#state{jid_set = JIDsSet, url_set = URLsSet, blocked_domains = maps:merge(BlockedDomains, set_to_map(SpamDomainsSet)), - whitelist_domains = set_to_map(WhitelistDomains, false)}} - catch - {Op, File, Reason} when Op == open; Op == read -> - Txt = format("Cannot ~s ~s for ~s: ~s", [Op, File, Host, format_error(Reason)]), - ?ERROR_MSG("~s", [Txt]), - {{error, Txt}, State} + whitelist_domains = set_to_map(WhitelistDomains, false)}}; + {config_error, ErrorText} -> + {{error, ErrorText}, State} end. set_to_map(Set) -> @@ -532,80 +516,18 @@ set_to_map(Set) -> set_to_map(Set, V) -> sets:fold(fun(K, M) -> M#{K => V} end, #{}, Set). --spec read_files(#{Type => filename()}) -> - #{jid => jid_set(), - url => url_set(), - Type => sets:set(binary())} - when Type :: atom(). -read_files(Files) -> - maps:map(fun(Type, Filename) -> read_file(Filename, line_parser(Type)) end, Files). - --spec line_parser(Type :: atom()) -> fun((binary()) -> binary()). -line_parser(jid) -> - fun parse_jid/1; -line_parser(url) -> - fun parse_url/1; -line_parser(_) -> - fun trim/1. - --spec read_file(filename(), fun((binary()) -> ljid() | url())) -> jid_set() | url_set(). -read_file(none, _ParseLine) -> - sets:new(); -read_file(File, ParseLine) -> - case file:open(File, [read, binary, raw, {read_ahead, 65536}]) of - {ok, Fd} -> - try - read_line(Fd, ParseLine, sets:new()) - catch - E -> - throw({read, File, E}) - after - ok = file:close(Fd) - end; - {error, Reason} -> - throw({open, File, Reason}) - end. - --spec read_line(file:io_device(), - fun((binary()) -> ljid() | url()), - jid_set() | url_set()) -> - jid_set() | url_set(). -read_line(Fd, ParseLine, Set) -> - case file:read_line(Fd) of - {ok, Line} -> - read_line(Fd, ParseLine, sets:add_element(ParseLine(Line), Set)); - {error, Reason} -> - throw(Reason); - eof -> - Set - end. - --spec parse_jid(binary()) -> ljid(). -parse_jid(S) -> - try jid:decode(trim(S)) of - #jid{} = JID -> - jid:remove_resource( - jid:tolower(JID)) - catch - _:{bad_jid, _} -> - throw({bad_jid, S}) - end. - --spec parse_url(binary()) -> url(). -parse_url(S) -> - URL = trim(S), - RE = <<"https?://\\S+$">>, - Options = [anchored, caseless, {capture, none}], - case re:run(URL, RE, Options) of - match -> - URL; - nomatch -> - throw({bad_url, S}) - end. - --spec trim(binary()) -> binary(). -trim(S) -> - re:replace(S, <<"\\s+$">>, <<>>, [{return, binary}]). +read_files(Host) -> + AccInitial = + #{jid => sets:new(), + url => sets:new(), + domains => sets:new(), + whitelist_domains => sets:new()}, + Files = + #{jid => gen_mod:get_module_opt(Host, ?MODULE, spam_jids_file), + url => gen_mod:get_module_opt(Host, ?MODULE, spam_urls_file), + domains => gen_mod:get_module_opt(Host, ?MODULE, spam_domains_file), + whitelist_domains => gen_mod:get_module_opt(Host, ?MODULE, whitelist_domains_file)}, + ejabberd_hooks:run_fold(antispam_get_lists, Host, AccInitial, [Files]). -spec get_proc_name(binary()) -> atom(). get_proc_name(Host) -> @@ -623,14 +545,6 @@ sets_equal(A, B) -> format(Format, Data) -> iolist_to_binary(io_lib:format(Format, Data)). --spec format_error(atom() | tuple()) -> binary(). -format_error({bad_jid, JID}) -> - <<"Not a valid JID: ", JID/binary>>; -format_error({bad_url, URL}) -> - <<"Not an HTTP(S) URL: ", URL/binary>>; -format_error(Reason) -> - list_to_binary(file:format_error(Reason)). - %%-------------------------------------------------------------------- %%| Caching @@ -792,16 +706,11 @@ try_call_by_host(Host, Call) -> reload_spam_filter_files(<<"global">>) -> for_all_hosts(fun reload_spam_filter_files/1, []); reload_spam_filter_files(Host) -> - LServer = jid:nameprep(Host), - Files = - #{domains => gen_mod:get_module_opt(LServer, ?MODULE, spam_domains_file), - jid => gen_mod:get_module_opt(LServer, ?MODULE, spam_jids_file), - url => gen_mod:get_module_opt(LServer, ?MODULE, spam_urls_file)}, - case try_call_by_host(Host, {reload_files, Files}) of + case try_call_by_host(Host, reload_spam_files) of {spam_filter, ok} -> ok; {spam_filter, {error, Txt}} -> - {error, binary_to_list(Txt)}; + {error, Txt}; {error, _R} = Error -> Error end. diff --git a/src/mod_antispam_files.erl b/src/mod_antispam_files.erl new file mode 100644 index 000000000..0981edbb2 --- /dev/null +++ b/src/mod_antispam_files.erl @@ -0,0 +1,182 @@ +%%%---------------------------------------------------------------------- +%%% File : mod_antispam_files.erl +%%% Author : Holger Weiss +%%% Author : Stefan Strigler +%%% Purpose : Filter spam messages based on sender JID and content +%%% Created : 31 Mar 2019 by Holger Weiss +%%% +%%% +%%% ejabberd, Copyright (C) 2019-2025 ProcessOne +%%% +%%% This program is free software; you can redistribute it and/or +%%% modify it under the terms of the GNU General Public License as +%%% published by the Free Software Foundation; either version 2 of the +%%% License, or (at your option) any later version. +%%% +%%% This program is distributed in the hope that it will be useful, +%%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +%%% General Public License for more details. +%%% +%%% You should have received a copy of the GNU General Public License along +%%% with this program; if not, write to the Free Software Foundation, Inc., +%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +%%% +%%%---------------------------------------------------------------------- + +%%| definitions +%% @format-begin + +-module(mod_antispam_files). + +-author('holger@zedat.fu-berlin.de'). +-author('stefan@strigler.de'). + +%% Exported +-export([init_files/1, terminate_files/1]). +% Hooks +-export([get_files_lists/2]). + +-include("ejabberd_commands.hrl"). +-include("logger.hrl"). +-include("mod_antispam.hrl"). +-include("translate.hrl"). + +-include_lib("xmpp/include/xmpp.hrl"). + +-type files_map() :: #{atom() => filename()}. +-type lists_map() :: + #{jid => jid_set(), + url => url_set(), + atom() => sets:set(binary())}. + +-define(COMMAND_TIMEOUT, timer:seconds(30)). +-define(DEFAULT_CACHE_SIZE, 10000). +-define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). +-define(HTTPC_TIMEOUT, timer:seconds(3)). + +%%-------------------------------------------------------------------- +%%| Exported + +init_files(Host) -> + ejabberd_hooks:add(antispam_get_lists, Host, ?MODULE, get_files_lists, 50). + +terminate_files(Host) -> + ejabberd_hooks:delete(antispam_get_lists, Host, ?MODULE, get_files_lists, 50). + +%%-------------------------------------------------------------------- +%%| Hooks + +-spec get_files_lists(lists_map(), files_map()) -> lists_map(). +get_files_lists(#{jid := AccJids, + url := AccUrls, + domains := AccDomains, + whitelist_domains := AccWhitelist} = + Acc, + Files) -> + try read_files(Files) of + #{jid := JIDsSet, + url := URLsSet, + domains := SpamDomainsSet, + whitelist_domains := WhitelistDomains} -> + Acc#{jid => sets:union(AccJids, JIDsSet), + url => sets:union(AccUrls, URLsSet), + domains => sets:union(AccDomains, SpamDomainsSet), + whitelist_domains => sets:union(AccWhitelist, WhitelistDomains)} + catch + {Op, File, Reason} when Op == open; Op == read -> + ErrorText = format("Error trying to ~s file ~s: ~s", [Op, File, format_error(Reason)]), + ?CRITICAL_MSG(ErrorText, []), + {stop, {config_error, ErrorText}} + end. + +%%-------------------------------------------------------------------- +%%| read_files + +-spec read_files(files_map()) -> lists_map(). +read_files(Files) -> + maps:map(fun(Type, Filename) -> read_file(Filename, line_parser(Type)) end, Files). + +-spec line_parser(Type :: atom()) -> fun((binary()) -> binary()). +line_parser(jid) -> + fun parse_jid/1; +line_parser(url) -> + fun parse_url/1; +line_parser(_) -> + fun trim/1. + +-spec read_file(filename(), fun((binary()) -> ljid() | url())) -> jid_set() | url_set(). +read_file(none, _ParseLine) -> + sets:new(); +read_file(File, ParseLine) -> + case file:open(File, [read, binary, raw, {read_ahead, 65536}]) of + {ok, Fd} -> + try + read_line(Fd, ParseLine, sets:new()) + catch + E -> + throw({read, File, E}) + after + ok = file:close(Fd) + end; + {error, Reason} -> + throw({open, File, Reason}) + end. + +-spec read_line(file:io_device(), + fun((binary()) -> ljid() | url()), + jid_set() | url_set()) -> + jid_set() | url_set(). +read_line(Fd, ParseLine, Set) -> + case file:read_line(Fd) of + {ok, Line} -> + read_line(Fd, ParseLine, sets:add_element(ParseLine(Line), Set)); + {error, Reason} -> + throw(Reason); + eof -> + Set + end. + +-spec parse_jid(binary()) -> ljid(). +parse_jid(S) -> + try jid:decode(trim(S)) of + #jid{} = JID -> + jid:remove_resource( + jid:tolower(JID)) + catch + _:{bad_jid, _} -> + throw({bad_jid, S}) + end. + +-spec parse_url(binary()) -> url(). +parse_url(S) -> + URL = trim(S), + RE = <<"https?://\\S+$">>, + Options = [anchored, caseless, {capture, none}], + case re:run(URL, RE, Options) of + match -> + URL; + nomatch -> + throw({bad_url, S}) + end. + +-spec trim(binary()) -> binary(). +trim(S) -> + re:replace(S, <<"\\s+$">>, <<>>, [{return, binary}]). + +%% Function copied from mod_antispam.erl +-spec format(io:format(), [term()]) -> binary(). +format(Format, Data) -> + iolist_to_binary(io_lib:format(Format, Data)). + +-spec format_error(atom() | tuple()) -> binary(). +format_error({bad_jid, JID}) -> + <<"Not a valid JID: ", JID/binary>>; +format_error({bad_url, URL}) -> + <<"Not an HTTP(S) URL: ", URL/binary>>; +format_error(Reason) -> + list_to_binary(file:format_error(Reason)). + +%%-------------------------------------------------------------------- + +%%| vim: set foldmethod=marker foldmarker=%%|,%%-: From 3d89c9199ccf775dea7ad8d3a66a7d56ccb5d5f8 Mon Sep 17 00:00:00 2001 From: Badlop Date: Thu, 26 Jun 2025 12:33:48 +0200 Subject: [PATCH 22/27] gen_mod: Add support to prepare module stopping before actually stopping any module Follows the reasoning of application:prep_stop, but applied to gen_mod: https://www.erlang.org/docs/28/apps/kernel/application.html#c:prep_stop/1 --- src/ejabberd_app.erl | 1 + src/gen_mod.erl | 40 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/ejabberd_app.erl b/src/ejabberd_app.erl index 5d15d98fd..a31d7b1a6 100644 --- a/src/ejabberd_app.erl +++ b/src/ejabberd_app.erl @@ -109,6 +109,7 @@ prep_stop(State) -> ejabberd_service:stop(), ejabberd_s2s:stop(), ejabberd_system_monitor:stop(), + gen_mod:prep_stop(), gen_mod:stop(), State. diff --git a/src/gen_mod.erl b/src/gen_mod.erl index c07e1d691..dc3ba3c56 100644 --- a/src/gen_mod.erl +++ b/src/gen_mod.erl @@ -27,7 +27,7 @@ -author('alexey@process-one.net'). -export([init/1, start_link/0, start_child/3, start_child/4, - stop_child/1, stop_child/2, stop/0, config_reloaded/0]). + stop_child/1, stop_child/2, prep_stop/0, stop/0, config_reloaded/0]). -export([start_module/2, stop_module/2, stop_module_keep_config/2, get_opt/2, set_opt/3, get_opt_hosts/1, is_equal_opt/3, get_module_opt/3, get_module_opts/2, get_module_opt_hosts/2, @@ -76,6 +76,7 @@ -callback start(binary(), opts()) -> ok | {ok, pid()} | {ok, [registration()]} | {error, term()}. +-callback prep_stop(binary()) -> any(). -callback stop(binary()) -> any(). -callback reload(binary(), opts(), opts()) -> ok | {ok, pid()} | {error, term()}. -callback mod_opt_type(atom()) -> econf:validator(). @@ -86,7 +87,7 @@ example => [string()] | [{binary(), [string()]}]}. -callback depends(binary(), opts()) -> [{module(), hard | soft}]. --optional_callbacks([mod_opt_type/1, reload/3]). +-optional_callbacks([mod_opt_type/1, reload/3, prep_stop/1]). -export_type([opts/0]). -export_type([db_type/0]). @@ -114,6 +115,10 @@ init([]) -> {read_concurrency, true}]), {ok, {{one_for_one, 10, 1}, []}}. +-spec prep_stop() -> ok. +prep_stop() -> + prep_stop_modules(). + -spec stop() -> ok. stop() -> ejabberd_hooks:delete(config_reloaded, ?MODULE, config_reloaded, 60), @@ -301,6 +306,21 @@ is_app_running(AppName) -> lists:keymember(AppName, 1, application:which_applications(Timeout)). +-spec prep_stop_modules() -> ok. +prep_stop_modules() -> + lists:foreach( + fun(Host) -> + prep_stop_modules(Host) + end, ejabberd_option:hosts()). + +-spec prep_stop_modules(binary()) -> ok. +prep_stop_modules(Host) -> + Modules = lists:reverse(loaded_modules_with_opts(Host)), + lists:foreach( + fun({Module, _Args}) -> + prep_stop_module_keep_config(Host, Module) + end, Modules). + -spec stop_modules() -> ok. stop_modules() -> lists:foreach( @@ -320,6 +340,22 @@ stop_modules(Host) -> stop_module(Host, Module) -> stop_module_keep_config(Host, Module). +-spec prep_stop_module_keep_config(binary(), atom()) -> error | ok. +prep_stop_module_keep_config(Host, Module) -> + ?DEBUG("Preparing to stop ~ts at ~ts", [Module, Host]), + try Module:prep_stop(Host) of + _ -> + ok + catch ?EX_RULE(error, undef, _St) -> + ok; + ?EX_RULE(Class, Reason, St) -> + StackTrace = ?EX_STACK(St), + ?ERROR_MSG("Failed to prepare stop module ~ts at ~ts:~n** ~ts", + [Module, Host, + misc:format_exception(2, Class, Reason, StackTrace)]), + error + end. + -spec stop_module_keep_config(binary(), atom()) -> error | ok. stop_module_keep_config(Host, Module) -> ?DEBUG("Stopping ~ts at ~ts", [Module, Host]), From 263e1f59f718d62b5dc56d3e0fc14f7ab059710c Mon Sep 17 00:00:00 2001 From: Badlop Date: Fri, 27 Jun 2025 16:38:46 +0200 Subject: [PATCH 23/27] Fix problem calling get_log_path when ejabberd is stopping When ejabberd is being stopped and some module calls ejabberd_logger:get_log_path(), application:load/1 crashes with error: ** Reason for termination == ** {terminating, [{application_controller,call,2, [{file,"application_controller.erl"},{line,511}]}, {application,load1,2,[{file,"application.erl"},{line,274}]}, {ejabberd_config,env_binary_to_list,2, [{file,"/home/git/ejabberd/src/ejabberd_config.erl"}, {line,343}]}, {ejabberd_logger,get_log_path,0, [{file,"/home/git/ejabberd/src/ejabberd_logger.erl"}, {line,55}]}, --- src/ejabberd_config.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ejabberd_config.erl b/src/ejabberd_config.erl index 89f89504d..a1bf043a6 100644 --- a/src/ejabberd_config.erl +++ b/src/ejabberd_config.erl @@ -340,7 +340,12 @@ may_hide_data(Data) -> -spec env_binary_to_list(atom(), atom()) -> {ok, any()} | undefined. env_binary_to_list(Application, Parameter) -> %% Application need to be loaded to allow setting parameters - application:load(Application), + case proplists:is_defined(Application, application:loaded_applications()) of + true -> + ok; + false -> + application:load(Application) + end, case application:get_env(Application, Parameter) of {ok, Val} when is_binary(Val) -> BVal = binary_to_list(Val), From b65c11daf67e9d6e8b8fd8d587dff35f3ee91ff3 Mon Sep 17 00:00:00 2001 From: Badlop Date: Thu, 26 Jun 2025 00:14:11 +0200 Subject: [PATCH 24/27] New predefined keyword: CONFIG_PATH --- src/ejabberd_config.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ejabberd_config.erl b/src/ejabberd_config.erl index a1bf043a6..29e491c0e 100644 --- a/src/ejabberd_config.erl +++ b/src/ejabberd_config.erl @@ -509,11 +509,15 @@ get_predefined_keywords(Host) -> [{<<"HOST">>, Host}] end, Home = misc:get_home(), + ConfigDirPath = + iolist_to_binary(filename:dirname( + ejabberd_config:path())), LogDirPath = iolist_to_binary(filename:dirname( ejabberd_logger:get_log_path())), HostList ++ [{<<"HOME">>, list_to_binary(Home)}, + {<<"CONFIG_PATH">>, ConfigDirPath}, {<<"LOG_PATH">>, LogDirPath}, {<<"SEMVER">>, ejabberd_option:version()}, {<<"VERSION">>, From c3f5083f15a9ad2df4c04cfa22cf03c3be5506ee Mon Sep 17 00:00:00 2001 From: Badlop Date: Fri, 27 Jun 2025 17:56:37 +0200 Subject: [PATCH 25/27] Use the new gen_mod:prep_stop/1 feature This fixes the problem when stopping the module with multiple vhosts: unsubscribing from a local pubsub requires mod_pubsub in that vhost running, but ejabberd stops mod_pubsub from a vhost before stopping mod_antispam in other vhost. --- src/mod_antispam.erl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index 500cb0244..1706f4377 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -35,6 +35,7 @@ %% gen_mod callbacks. -export([start/2, + prep_stop/1, stop/1, reload/3, depends/2, @@ -104,6 +105,13 @@ start(Host, Opts) -> end, gen_mod:start_child(?MODULE, Host, Opts). +-spec prep_stop(binary()) -> ok | {error, any()}. +prep_stop(Host) -> + case try_call_by_host(Host, prepare_stop) of + ready_to_stop -> + ok + end. + -spec stop(binary()) -> ok | {error, any()}. stop(Host) -> case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of @@ -317,6 +325,14 @@ handle_call({is_blocked_domain, Domain}, {reply, maps:get(Domain, maps:merge(BlockedDomains, WhitelistDomains), false) =/= false, State}; +handle_call(prepare_stop, + _From, + #state{host = Host, + rtbl_host = RTBLHost, + rtbl_domains_node = RTBLDomainsNode} = + State) -> + mod_antispam_rtbl:unsubscribe(RTBLHost, RTBLDomainsNode, Host), + {reply, ready_to_stop, State}; handle_call(Request, From, State) -> ?ERROR_MSG("Got unexpected request from ~p: ~p", [From, Request]), {noreply, State}. From 5f293cb1e0f163117e7a8c3ab1fe300ea82806dc Mon Sep 17 00:00:00 2001 From: Badlop Date: Mon, 23 Jun 2025 09:58:52 +0200 Subject: [PATCH 26/27] Document more options --- src/mod_antispam.erl | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index 1706f4377..9baaeffc1 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -172,7 +172,14 @@ mod_options(_Host) -> {whitelist_domains_file, none}]. mod_doc() -> - #{desc => ?T("Reads from text file and RTBL, filters stanzas and writes dump file."), + #{desc => + ?T("Filter spam messages and subscription requests received from " + "remote servers based on " + "https://xmppbl.org/[Real-Time Block Lists (RTBL)], " + "lists of known spammer JIDs and/or URLs mentioned in spam messages. " + "Traffic classified as spam is rejected with an error " + "(and an '[info]' message is logged) unless the sender " + "is subscribed to the recipient's presence."), note => "added in 25.xx", opts => [{access_spam, @@ -183,6 +190,16 @@ mod_doc() -> "spam messages aren't rejected for that recipient. " "The default value is 'none', which means that all recipients " "are subject to spam filtering verification.")}}, + {cache_size, + #{value => "pos_integer()", + desc => + ?T("Maximum number of JIDs that will be cached due to sending spam URLs. " + "If that limit is exceeded, the least recently used " + "entries are removed from the cache. " + "Setting this option to '0' disables the caching feature. " + "Note that separate caches are used for each virtual host, " + " and that the caches aren't distributed across cluster nodes. " + "The default value is '10000'.")}}, {spam_domains_file, #{value => ?T("none | Path"), desc => @@ -197,9 +214,11 @@ mod_doc() -> #{value => ?T("false | true | Path"), desc => ?T("Path to the file to store blocked messages. " - "Use an absolute path, or the '@LOG_PATH@' macro to store logs " + "Use an absolute path, or the '@LOG_PATH@' " + "https://docs.ejabberd.im/admin/configuration/file-format/#predefined-keywords[predefined keyword] " + "to store logs " "in the same place that the other ejabberd log files. " - "If set to 'false', does not dump stanzas, this is the default. " + "If set to 'false', it doesn't dump stanzas, which is the default. " "If set to 'true', it stores in '\"@LOG_PATH@/spam_dump_@HOST@.log\"'.")}}, {spam_jids_file, #{value => ?T("none | Path"), @@ -228,12 +247,13 @@ mod_doc() -> ?T("Path to a file containing a list of " "domains to whitelist from being blocked, one per line. " "If either it is in 'spam_domains_file' or more realistically " - "in a domain sent by a RTBL host (see option 'rtbl_host') " + "in a domain sent by a RTBL host (see option 'rtbl_services') " "then this domain will be ignored and stanzas from there won't be blocked. " "The default value is 'none'.")}}], example => ["modules:", " mod_antispam:", + " spam_jids_file: \"@CONFIG_PATH@/spam_jids.txt\"", " spam_dump_file: \"@LOG_PATH@/spam/host-@HOST@.log\""]}. %%-------------------------------------------------------------------- From 5e93725044c54e4f440276a6f7e7daf9902ca36a Mon Sep 17 00:00:00 2001 From: Badlop Date: Fri, 27 Jun 2025 17:56:58 +0200 Subject: [PATCH 27/27] Replace options rtbl_host and rtbl_domains_node with rtbl_services --- include/mod_antispam.hrl | 10 +++ src/mod_antispam.erl | 67 ++++++++++++++++---- src/mod_antispam_files.erl | 1 - src/mod_antispam_rtbl.erl | 7 +- test/antispam_tests.erl | 7 +- test/ejabberd_SUITE_data/ejabberd.mnesia.yml | 3 +- test/ejabberd_SUITE_data/ejabberd.redis.yml | 3 +- 7 files changed, 76 insertions(+), 22 deletions(-) diff --git a/include/mod_antispam.hrl b/include/mod_antispam.hrl index 7f681046e..c30f24620 100644 --- a/include/mod_antispam.hrl +++ b/include/mod_antispam.hrl @@ -24,3 +24,13 @@ -type filename() :: binary() | none | false. -type jid_set() :: sets:set(ljid()). -type url_set() :: sets:set(url()). + +-define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). + +-record(rtbl_service, + {host = none :: binary() | none, + node = ?DEFAULT_RTBL_DOMAINS_NODE :: binary(), + subscribed = false :: boolean(), + retry_timer = undefined :: reference() | undefined}). + +-type rtbl_service() :: #rtbl_service{}. diff --git a/src/mod_antispam.erl b/src/mod_antispam.erl index 9baaeffc1..0e3a1b8a7 100644 --- a/src/mod_antispam.erl +++ b/src/mod_antispam.erl @@ -51,6 +51,8 @@ terminate/2, code_change/3]). +-export([get_rtbl_services_option/1]). + %% ejabberd_commands callbacks. -export([add_blocked_domain/2, add_to_spam_filter_cache/2, @@ -87,7 +89,6 @@ -type state() :: #state{}. -define(COMMAND_TIMEOUT, timer:seconds(30)). --define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). -define(DEFAULT_CACHE_SIZE, 10000). %% @format-begin @@ -137,12 +138,14 @@ mod_opt_type(access_spam) -> econf:acl(); mod_opt_type(cache_size) -> econf:pos_int(unlimited); -mod_opt_type(rtbl_host) -> - econf:either( - econf:enum([none]), econf:host()); -mod_opt_type(rtbl_domains_node) -> - econf:non_empty( - econf:binary()); +mod_opt_type(rtbl_services) -> + econf:list( + econf:either( + econf:binary(), + econf:map( + econf:binary(), + econf:map( + econf:enum([spam_source_domains_node]), econf:binary())))); mod_opt_type(spam_domains_file) -> econf:either( econf:enum([none]), econf:file()); @@ -159,12 +162,11 @@ mod_opt_type(whitelist_domains_file) -> econf:either( econf:enum([none]), econf:file()). --spec mod_options(binary()) -> [{atom(), any()}]. +-spec mod_options(binary()) -> [{rtbl_services, [tuple()]} | {atom(), any()}]. mod_options(_Host) -> [{access_spam, none}, {cache_size, ?DEFAULT_CACHE_SIZE}, - {rtbl_domains_node, ?DEFAULT_RTBL_DOMAINS_NODE}, - {rtbl_host, none}, + {rtbl_services, []}, {spam_domains_file, none}, {spam_dump_file, false}, {spam_jids_file, none}, @@ -200,6 +202,21 @@ mod_doc() -> "Note that separate caches are used for each virtual host, " " and that the caches aren't distributed across cluster nodes. " "The default value is '10000'.")}}, + {rtbl_services, + #{value => ?T("[Service]"), + example => + ["rtbl_services:", + " - pubsub.server1.localhost:", + " spam_source_domains_node: actual_custom_pubsub_node"], + desc => + ?T("Query a RTBL service to get domains to block, as provided by " + "https://xmppbl.org/[xmppbl.org]. " + "Please note right now this option only supports one service in that list. " + "For blocking spam and abuse on MUC channels, please use _`mod_muc_rtbl`_ for now. " + "If only the host is provided, the default node names will be assumed. " + "If the node name is different than 'spam_source_domains', " + "you can setup the custom node name with the option 'spam_source_domains_node'. " + "The default value is an empty list of services.")}}, {spam_domains_file, #{value => ?T("none | Path"), desc => @@ -253,6 +270,8 @@ mod_doc() -> example => ["modules:", " mod_antispam:", + " rtbl_services:", + " - xmppbl.org", " spam_jids_file: \"@CONFIG_PATH@/spam_jids.txt\"", " spam_dump_file: \"@LOG_PATH@/spam/host-@HOST@.log\""]}. @@ -274,8 +293,7 @@ init([Host, Opts]) -> mod_antispam_rtbl, pubsub_event_handler, 50), - RTBLHost = gen_mod:get_opt(rtbl_host, Opts), - RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, Opts), + [#rtbl_service{host = RTBLHost, node = RTBLDomainsNode}] = get_rtbl_services_option(Opts), mod_antispam_filter:init_filtering(Host), InitState = #state{host = Host, @@ -384,8 +402,8 @@ handle_cast({reload, NewOpts, OldOpts}, end, ok = mod_antispam_rtbl:unsubscribe(OldRTBLHost, OldRTBLDomainsNode, Host), {_Result, State3} = reload_files(State2#state{blocked_domains = #{}}), - RTBLHost = gen_mod:get_opt(rtbl_host, NewOpts), - RTBLDomainsNode = gen_mod:get_opt(rtbl_domains_node, NewOpts), + [#rtbl_service{host = RTBLHost, node = RTBLDomainsNode}] = + get_rtbl_services_option(NewOpts), ok = mod_antispam_rtbl:request_blocked_domains(RTBLHost, RTBLDomainsNode, Host), {noreply, State3#state{rtbl_host = RTBLHost, rtbl_domains_node = RTBLDomainsNode}}; handle_cast({update_blocked_domains, NewItems}, @@ -565,6 +583,27 @@ read_files(Host) -> whitelist_domains => gen_mod:get_module_opt(Host, ?MODULE, whitelist_domains_file)}, ejabberd_hooks:run_fold(antispam_get_lists, Host, AccInitial, [Files]). +get_rtbl_services_option(Host) when is_binary(Host) -> + get_rtbl_services_option(gen_mod:get_module_opts(Host, ?MODULE)); +get_rtbl_services_option(Opts) when is_map(Opts) -> + Services = gen_mod:get_opt(rtbl_services, Opts), + case length(Services) =< 1 of + true -> + ok; + false -> + ?WARNING_MSG("Option rtbl_services only supports one service, but several " + "were configured. Will use only first one", + []) + end, + case Services of + [] -> + [#rtbl_service{}]; + [Host | _] when is_binary(Host) -> + [#rtbl_service{host = Host, node = ?DEFAULT_RTBL_DOMAINS_NODE}]; + [[{Host, [{spam_source_domains_node, Node}]}] | _] -> + [#rtbl_service{host = Host, node = Node}] + end. + -spec get_proc_name(binary()) -> atom(). get_proc_name(Host) -> gen_mod:get_module_proc(Host, ?MODULE). diff --git a/src/mod_antispam_files.erl b/src/mod_antispam_files.erl index 0981edbb2..0362a1b72 100644 --- a/src/mod_antispam_files.erl +++ b/src/mod_antispam_files.erl @@ -52,7 +52,6 @@ -define(COMMAND_TIMEOUT, timer:seconds(30)). -define(DEFAULT_CACHE_SIZE, 10000). --define(DEFAULT_RTBL_DOMAINS_NODE, <<"spam_source_domains">>). -define(HTTPC_TIMEOUT, timer:seconds(3)). %%-------------------------------------------------------------------- diff --git a/src/mod_antispam_rtbl.erl b/src/mod_antispam_rtbl.erl index 246b59dfa..df5bac591 100644 --- a/src/mod_antispam_rtbl.erl +++ b/src/mod_antispam_rtbl.erl @@ -78,7 +78,7 @@ request_blocked_domains(RTBLHost, RTBLDomainsNode, From) -> -spec parse_blocked_domains(stanza()) -> #{binary() => any()} | undefined. parse_blocked_domains(#iq{to = #jid{lserver = LServer}, type = result} = IQ) -> ?DEBUG("parsing iq-result items: ~p", [IQ]), - RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), + [#rtbl_service{node = RTBLDomainsNode}] = mod_antispam:get_rtbl_services_option(LServer), case xmpp:get_subtag(IQ, #pubsub{}) of #pubsub{items = #ps_items{node = RTBLDomainsNode, items = Items}} -> ?DEBUG("Got items:~n~p", [Items]), @@ -89,7 +89,7 @@ parse_blocked_domains(#iq{to = #jid{lserver = LServer}, type = result} = IQ) -> -spec parse_pubsub_event(stanza()) -> #{binary() => any()}. parse_pubsub_event(#message{to = #jid{lserver = LServer}} = Msg) -> - RTBLDomainsNode = gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_domains_node), + [#rtbl_service{node = RTBLDomainsNode}] = mod_antispam:get_rtbl_services_option(LServer), case xmpp:get_subtag(Msg, #ps_event{}) of #ps_event{items = #ps_items{node = RTBLDomainsNode, @@ -130,7 +130,8 @@ pubsub_event_handler(#message{from = FromJid, Msg) -> ?DEBUG("Got RTBL message:~n~p", [Msg]), From = jid:encode(FromJid), - case gen_mod:get_module_opt(LServer, ?SERVICE_MODULE, rtbl_host) of + [#rtbl_service{host = RTBLHost}] = mod_antispam:get_rtbl_services_option(LServer), + case RTBLHost of From -> ParsedItems = parse_pubsub_event(Msg), Proc = gen_mod:get_module_proc(LServer, ?SERVICE_MODULE), diff --git a/test/antispam_tests.erl b/test/antispam_tests.erl index f60872913..8d7bd2472 100644 --- a/test/antispam_tests.erl +++ b/test/antispam_tests.erl @@ -31,6 +31,7 @@ disconnect/1, put_event/2, get_event/1, peer_muc_jid/1, my_muc_jid/1, get_features/2, set_opt/3]). -include("suite.hrl"). +-include("mod_antispam.hrl"). %% @format-begin @@ -166,7 +167,8 @@ rtbl_domains(Config) -> RTBLDomainsNode = <<"spam_source_domains">>, OldOpts = gen_mod:get_module_opts(Host, mod_antispam), NewOpts = - maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}), + maps:merge(OldOpts, + #{rtbl_services => [#rtbl_service{host = RTBLHost, node = RTBLDomainsNode}]}), Owner = jid:make(?config(user, Config), ?config(server, Config), <<>>), {result, _} = mod_pubsub:create_node(RTBLHost, @@ -210,7 +212,8 @@ rtbl_domains_whitelisted(Config) -> RTBLDomainsNode = <<"spam_source_domains">>, OldOpts = gen_mod:get_module_opts(Host, mod_antispam), NewOpts = - maps:merge(OldOpts, #{rtbl_host => RTBLHost, rtbl_domains_node => RTBLDomainsNode}), + maps:merge(OldOpts, + #{rtbl_services => [#rtbl_service{host = RTBLHost, node = RTBLDomainsNode}]}), Owner = jid:make(?config(user, Config), ?config(server, Config), <<>>), {result, _} = mod_pubsub:create_node(RTBLHost, diff --git a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml index ae78645ae..56fdf5e6e 100644 --- a/test/ejabberd_SUITE_data/ejabberd.mnesia.yml +++ b/test/ejabberd_SUITE_data/ejabberd.mnesia.yml @@ -7,7 +7,8 @@ define_macro: db_type: internal access: local mod_antispam: - rtbl_host: pubsub.mnesia.localhost + rtbl_services: + - "pubsub.mnesia.localhost" spam_jids_file: spam_jids.txt spam_domains_file: spam_domains.txt spam_urls_file: spam_urls.txt diff --git a/test/ejabberd_SUITE_data/ejabberd.redis.yml b/test/ejabberd_SUITE_data/ejabberd.redis.yml index 8ec927e86..fb1ba435f 100644 --- a/test/ejabberd_SUITE_data/ejabberd.redis.yml +++ b/test/ejabberd_SUITE_data/ejabberd.redis.yml @@ -8,7 +8,8 @@ define_macro: db_type: internal access: local mod_antispam: - rtbl_host: pubsub.redis.localhost + rtbl_services: + - "pubsub.redis.localhost" spam_jids_file: spam_jids.txt spam_domains_file: spam_domains.txt spam_urls_file: spam_urls.txt