changeset 278:368572c57013

add limits on unique ip addresses per hour per authenticated user
author Carl Byington <carl@five-ten-sg.com>
date Tue, 17 Dec 2013 15:35:23 -0800
parents 7163e9b04bdb
children 3d894d09c198
files ChangeLog NEWS configure.in dnsbl.conf src/context.cpp src/context.h src/dnsbl.cpp tld.conf xml/dnsbl.in
diffstat 9 files changed, 239 insertions(+), 147 deletions(-) [+]
line wrap: on
line diff
--- a/ChangeLog	Wed Dec 11 22:57:06 2013 -0800
+++ b/ChangeLog	Tue Dec 17 15:35:23 2013 -0800
@@ -1,3 +1,6 @@
+6.37 2013-12-16
+    Add unique ip connection limits per authenticated id or email address.
+
 6.36 2013-09-09
     Code cleanup, increase minimum hostname length for uribl checking.
 
--- a/NEWS	Wed Dec 11 22:57:06 2013 -0800
+++ b/NEWS	Tue Dec 17 15:35:23 2013 -0800
@@ -1,3 +1,4 @@
+6.37 2013-12-16 Add unique ip connection limits per authenticated id or email address.
 6.36 2013-09-09 Code cleanup, increase minimum hostname length for uribl checking.
 6.35 2013-09-09 Use mozilla prefix list for tld checking. Enable surbl/uribl/dbl rhs lists.
 6.34 2013-05-22 Add require_rdns checking.
--- a/configure.in	Wed Dec 11 22:57:06 2013 -0800
+++ b/configure.in	Tue Dec 17 15:35:23 2013 -0800
@@ -1,6 +1,6 @@
 
 AC_PREREQ(2.59)
-AC_INIT(dnsbl,6.36,carl@five-ten-sg.com)
+AC_INIT(dnsbl,6.37,carl@five-ten-sg.com)
 AC_CONFIG_SRCDIR([config.h.in])
 AC_CONFIG_HEADER([config.h])
 
--- a/dnsbl.conf	Wed Dec 11 22:57:06 2013 -0800
+++ b/dnsbl.conf	Tue Dec 17 15:35:23 2013 -0800
@@ -22,21 +22,23 @@
         dcc_bulk_threshold  50;
     };
 
-    // backscatter prevention - don't send bounces for mail that we accepted but could not forward
+    // backscatter prevention - do not send bounces for mail that we accepted but could not forward
     // we only send bounces to our own customers
     env_from unknown {
         "<>"    black;
     };
 
-    // hourly recipient rate limit by smtp auth client id,
-    // or unauthenticated mail from address
-    // default hourly limit is 30
-    // daily limits are 4 times the hourly limit
-    rate_limit 30 4 { // default
-        #fred 100;   // override default limits
-        #joe  10;    // ""
-        #"sam@somedomain.tld"  500;
-        #"@otherdomain.tld"    100;
+    // hourly recipient rate limit by smtp auth client id, or unauthenticated mail from address
+    // hourly unique ip addresses  by smtp auth client id, or unauthenticated mail from address
+    // default hourly rate limit is 30
+    // daily rate limits are 4 times the hourly limit
+    // default hourly unique ip addresses is 5
+    // daily unique ip addresses are 4 times the hourly limit
+    rate_limit 30 4 5 4 { // default
+        fred 100 10;   // override default limits
+        joe  10  2;    // ""
+        "sam@somedomain.tld"  500 2;
+        "@otherdomain.tld"    100 2;
     };
 };
 
@@ -52,8 +54,9 @@
 
     content on {
         filter    sbl-xbl.spamhaus.org        "Mail containing %s rejected - sbl; see http://www.spamhaus.org/query/bl?ip=%s";
-        uribl     multi.surbl.org             "Mail containing %s rejected - surbl; see http://www.rulesemporium.com/cgi-bin/uribl.cgi?bl0=1&domain0=%s";
+        uribl     multi.surbl.org             "Mail containing %s rejected - surbl; see http://www.surbl.org/surbl-analysis?d=%s";
         #uribl    multi.uribl.com             "Mail containing %s rejected - uribl; see http://l.uribl.com/?d=%s";
+        #uribl    dbl.spamhaus.org            "Mail containing %s rejected - dbl; see http://www.spamhaus.org/query/domain?domain=%s";
         ignore    { include "hosts-ignore.conf"; };
         tld       { include "tld.conf"; };
         html_tags { include "html-tags.conf"; };
--- a/src/context.cpp	Wed Dec 11 22:57:06 2013 -0800
+++ b/src/context.cpp	Tue Dec 17 15:35:23 2013 -0800
@@ -733,8 +733,10 @@
     dcc_bulk_threshold  = (parent) ? parent->dcc_bulk_threshold : 0;
     dnsbl_list_parsed   = false;
     dnswl_list_parsed   = false;
-    default_rcpt_rate   = 36000;        // 10 per second
-    rcpt_daily_multiple = 3;
+    default_rate_limit      = 36000;        // 10 per second
+    default_address_limit   = 10;
+    daily_rate_multiple     = 3;
+    daily_address_multiple  = 3;
 }
 
 
@@ -845,20 +847,32 @@
 }
 
 
-int CONTEXT::find_rate(const char *user) {
-    if (rcpt_per_hour.empty()) return default_rcpt_rate;
-    rcpt_rates::iterator i = rcpt_per_hour.find(user);  // look for authen id, or sender user@email limiting
+int CONTEXT::find_rate_limit(const char *user) {
+    if (rcpt_per_hour.empty()) return default_rate_limit;
+    rates::iterator i = rcpt_per_hour.find(user);       // look for authen id, or sender user@email limiting
     if (i != rcpt_per_hour.end()) return (*i).second;   // found authen id, or user@email limiting
     const char *f = strchr(user, '@');
-    if (!f) return default_rcpt_rate;
+    if (!f) return default_rate_limit;
     i = rcpt_per_hour.find(f);                          // look for @domain limiting
     if (i != rcpt_per_hour.end()) return (*i).second;   // found @domain limiting
-    return default_rcpt_rate;
+    return default_rate_limit;
+}
+
+
+int CONTEXT::find_address_limit(const char *user) {
+    if (addresses_per_hour.empty()) return default_address_limit;
+    rates::iterator i = addresses_per_hour.find(user);      // look for authen id, or sender user@email limiting
+    if (i != addresses_per_hour.end()) return (*i).second;  // found authen id, or user@email limiting
+    const char *f = strchr(user, '@');
+    if (!f) return default_address_limit;
+    i = addresses_per_hour.find(f);                         // look for @domain limiting
+    if (i != addresses_per_hour.end()) return (*i).second;  // found @domain limiting
+    return default_address_limit;
 }
 
 
 bool CONTEXT::is_unauthenticated_limited(const char *user) {
-    rcpt_rates::iterator i = rcpt_per_hour.find(user);  // look for sender user@email limiting
+    rates::iterator i = rcpt_per_hour.find(user);       // look for sender user@email limiting
     if (i != rcpt_per_hour.end()) return true;          // found user@email limiting
     const char *f = strchr(user, '@');
     if (!f) return false;
@@ -1211,11 +1225,13 @@
     printf("%s     }; \n", indent);
 
     if (isdefault) {
-        printf("%s     rate_limit %d %d { \n", indent, default_rcpt_rate, rcpt_daily_multiple);
-        for (rcpt_rates::iterator j=rcpt_per_hour.begin(); j!=rcpt_per_hour.end(); j++) {
+        printf("%s     rate_limit %d %d %d %d { \n", indent, default_rate_limit, daily_rate_multiple, default_address_limit, daily_address_multiple);
+        for (rates::iterator j=rcpt_per_hour.begin(); j!=rcpt_per_hour.end(); j++) {
             const char *u = (*j).first;
             int         l = (*j).second;
-            printf("%s         \"%s\" \t%d; \n", indent, u, l);
+            rates::iterator k = addresses_per_hour.find(u);
+            int             a = (k==addresses_per_hour.end()) ? default_address_limit : (*k).second;
+            printf("%s         \"%s\" \t%d %d; \n", indent, u, l, a);
         }
         printf("%s     }; \n", indent);
     }
@@ -1753,25 +1769,18 @@
 //
 bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me);
 bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
-    const char *def = tok.next();
-    tok.push(def);
-    if (def != token_lbrace) me.set_default_rate(tok.nextint());
-
-    def = tok.next();
-    tok.push(def);
-    if (def != token_lbrace) me.set_daily_multiple(tok.nextint());
-
+    me.set_default_rate_limit(tok.nextint());
+    me.set_daily_rate_multiple(tok.nextint());
+    me.set_default_address_limit(tok.nextint());
+    me.set_daily_address_multiple(tok.nextint());
     if (!tsa(tok, token_lbrace)) return false;
     while (true) {
         const char *have = tok.next();
         if (!have) break;
         if (have == token_rbrace) break;
-        if (have == token_semi) {
-            // optional separators
-        }
-        else {
-            me.add_rate(have, tok.nextint());
-        }
+        me.add_rate_limit(have, tok.nextint());
+        me.add_address_limit(have, tok.nextint());
+        if (!tsa(tok, token_semi)) return false;
     }
     return tsa(tok, token_semi);
 }
--- a/src/context.h	Wed Dec 11 22:57:06 2013 -0800
+++ b/src/context.h	Tue Dec 17 15:35:23 2013 -0800
@@ -26,6 +26,7 @@
 typedef map<const char *, const char *, ltstr>  string_map;
 typedef set<int>                                int_set;
 typedef set<int32_t>                            int32_t_set;
+typedef int32_t_set *                           int32_t_set_p;
 typedef list<SMTP *>                            smtp_list;
 typedef DNSBL *                                 DNSBLP;
 typedef DNSWL *                                 DNSWLP;
@@ -40,7 +41,8 @@
 typedef list<CONTEXTP>                          context_list;
 typedef map<const char *, CONTEXTP, ltstr>      context_map;
 typedef map<const char *, int32_t, ltstr>       ns_mapper;  // name to ipv4 address
-typedef map<const char *, int, ltstr>           rcpt_rates;
+typedef map<const char *, int, ltstr>           rates;
+typedef map<const char *, int32_t_set_p, ltstr> auth_addresses;
 typedef map<const char *, time_t,  ltstr>       autowhite_sent;
 typedef map<const char *, VERIFYP, ltstr>       verify_map;
 typedef map<const char *, WHITELISTERP, ltstr>  whitelister_map;
@@ -181,9 +183,12 @@
     dnswlp_map      dnswl_names;        // name to dnswl mapping for lists that are available in this context and children
     dnswlp_list     dnswl_list;         // list of dnswls to be used in this context
     bool            dnswl_list_parsed;  // true iff we have actually parsed a dnswl_list
-    int             default_rcpt_rate;  // if not specified per user
-    int             rcpt_daily_multiple;// daily multiplier applied to hourly rate
-    rcpt_rates      rcpt_per_hour;      // per user limits on number of recipients per hour
+    int             default_rate_limit;     // if not specified per user
+    int             default_address_limit;  // if not specified per user
+    int             daily_rate_multiple;    // daily multiplier applied to hourly rate
+    int             daily_address_multiple; // daily multiplier applied to hourly rate
+    rates           rcpt_per_hour;          // per user limits on number of recipients per hour
+    rates           addresses_per_hour;     // per user limits on number of unique ip address connections per hour
 
 
 public:
@@ -207,12 +212,17 @@
     const char* get_autowhite()                             {return autowhite_file;};
     WHITELISTERP find_autowhite(const char *from, const char *to);
 
-    void        set_default_rate(int limit)                 {default_rcpt_rate   = limit;};
-    void        set_daily_multiple(int multiple)            {rcpt_daily_multiple = multiple;};
-    void        add_rate(const char *user, int limit)       {rcpt_per_hour[user] = limit;};
-    int         find_rate(const char *user);
+    void        set_default_rate_limit(int limit)               {default_rate_limit = limit;};
+    void        set_default_address_limit(int limit)            {default_address_limit = limit;};
+    void        set_daily_rate_multiple(int multiple)           {daily_rate_multiple = multiple;};
+    void        set_daily_address_multiple(int multiple)        {daily_address_multiple = multiple;};
+    void        add_rate_limit(const char *user, int limit)     {rcpt_per_hour[user] = limit;};
+    void        add_address_limit(const char *user, int limit)  {addresses_per_hour[user] = limit;};
+    int         find_rate_limit(const char *user);              // recipients per hour/day
+    int         find_address_limit(const char *user);           // unique ip address connections per hour/day
     bool        is_unauthenticated_limited(const char *user);
-    int         get_daily_multiple()                        {return rcpt_daily_multiple;};
+    int         get_daily_rate_multiple()                       {return daily_rate_multiple;};
+    int         get_daily_address_multiple()                    {return daily_address_multiple;};
 
     void        add_to(const char *to)                            {env_to.insert(to);};
     void        add_from(const char *from, const char *status)    {env_from[from] = status;};
--- a/src/dnsbl.cpp	Wed Dec 11 22:57:06 2013 -0800
+++ b/src/dnsbl.cpp	Tue Dec 17 15:35:23 2013 -0800
@@ -114,8 +114,10 @@
 time_t      last_error_time     = 0;
 int         resolver_sock_count = 0;        // protected with fd_pool_mutex
 int         resolver_pool_size  = 0;        // protected with fd_pool_mutex
-rcpt_rates  rcpt_hourly_counts;             // protected with rate_mutex
-rcpt_rates  rcpt_daily_counts;              // protected with rate_mutex
+rates       rcpt_hourly_counts;             // protected with rate_mutex
+rates       rcpt_daily_counts;              // protected with rate_mutex
+auth_addresses auth_hourly_addresses;       // protected with rate_mutex
+auth_addresses auth_daily_addresses;        // protected with rate_mutex
 
 
 struct ns_map {
@@ -167,7 +169,7 @@
 void incr_rcpt_count(const char *user, int &hourly, int &daily);
 void incr_rcpt_count(const char *user, int &hourly, int &daily) {
     pthread_mutex_lock(&rate_mutex);
-        rcpt_rates::iterator i = rcpt_hourly_counts.find(user);
+        rates::iterator i = rcpt_hourly_counts.find(user);
         hourly = 1;
         if (i == rcpt_hourly_counts.end()) {
             user = strdup(user);
@@ -177,7 +179,7 @@
             hourly = ++((*i).second);
         }
 
-        rcpt_rates::iterator j = rcpt_daily_counts.find(user);
+        rates::iterator j = rcpt_daily_counts.find(user);
         daily = 1;
         if (j == rcpt_daily_counts.end()) {
             user = strdup(user);
@@ -189,6 +191,36 @@
     pthread_mutex_unlock(&rate_mutex);
 }
 
+
+void add_auth_address(const char *user, int &hourly, int &daily, int32_t ip);
+void add_auth_address(const char *user, int &hourly, int &daily, int32_t ip) {
+    pthread_mutex_lock(&rate_mutex);
+        auth_addresses::iterator ii = auth_hourly_addresses.find(user);
+        if (ii == auth_hourly_addresses.end()) {
+            auth_hourly_addresses[user] = new int32_t_set;
+            auth_hourly_addresses[user]->insert(ip);
+            hourly = 1;
+        }
+        else {
+            int32_t_set::iterator i = ((*ii).second)->find(ip);
+            if (i == ((*ii).second)->end()) ((*ii).second)->insert(ip);
+            hourly = ((*ii).second)->size();
+        }
+
+        auth_addresses::iterator jj = auth_daily_addresses.find(user);
+        if (jj == auth_daily_addresses.end()) {
+            auth_daily_addresses[user] = new int32_t_set;
+            auth_daily_addresses[user]->insert(ip);
+            daily = 1;
+        }
+        else {
+            int32_t_set::iterator i = ((*jj).second)->find(ip);
+            if (i == ((*jj).second)->end()) ((*jj).second)->insert(ip);
+            daily = ((*jj).second)->size();
+        }
+    pthread_mutex_unlock(&rate_mutex);
+}
+
 ////////////////////////////////////////////////
 // helper to discard the strings held by a context_map
 //
@@ -1135,12 +1167,31 @@
 sfsistat mlfi_envfrom(SMFICTX *ctx, char **from)
 {
     mlfiPriv &priv     = *MLFIPRIV;
+    CONFIG &dc         = *priv.pc;
     priv.mailaddr      = to_lower_string(from[0]);
     priv.queueid       = strdup(smfi_getsymval(ctx, (char*)"i"));
     priv.authenticated = smfi_getsymval(ctx, (char*)"{auth_authen}");
     priv.client_name   = smfi_getsymval(ctx, (char*)"_");
     if (!priv.helo)         priv.helo          = strdup("unknown");
     if (priv.authenticated) priv.authenticated = strdup(priv.authenticated);
+    {
+        const char *uid = (priv.authenticated) ? priv.authenticated : priv.mailaddr;
+        if (priv.authenticated || dc.default_context->is_unauthenticated_limited(priv.mailaddr)) {
+            int hourly, daily;
+            add_auth_address(uid, hourly, daily, priv.ip);
+            int h_limit = dc.default_context->find_address_limit(uid);
+            int d_limit = dc.default_context->get_daily_address_multiple() * h_limit;
+            if (debug_syslog > 1) {
+                char msg[maxlen];
+                snprintf(msg, sizeof(msg), "connect for %s (%d %d addresses, %d %d limits)", uid, hourly, daily, h_limit, d_limit);
+                my_syslog(&priv, msg);
+            }
+            if ((hourly > h_limit) || (daily > d_limit)){
+                //smfi_setreply(ctx, (char*)"550", (char*)"5.7.1", (char*)"unique connection address limit exceeded");
+                //return SMFIS_REJECT;
+            }
+        }
+    }
     if (priv.client_name) {
         priv.client_name = strdup(priv.client_name);
         const char *p = strstr(priv.client_name, " [");
@@ -1211,8 +1262,8 @@
     if (priv.authenticated) {
         int hourly, daily;
         incr_rcpt_count(priv.authenticated, hourly, daily);
-        int h_limit = dc.default_context->find_rate(priv.authenticated);
-        int d_limit = dc.default_context->get_daily_multiple() * h_limit;
+        int h_limit = dc.default_context->find_rate_limit(priv.authenticated);
+        int d_limit = dc.default_context->get_daily_rate_multiple() * h_limit;
         if (debug_syslog > 1) {
             char msg[maxlen];
             snprintf(msg, sizeof(msg), "authenticated id %s (%d %d recipients, %d %d limits)", priv.authenticated, hourly, daily, h_limit, d_limit);
@@ -1295,8 +1346,8 @@
     if (!priv.authenticated && dc.default_context->is_unauthenticated_limited(priv.mailaddr)) {
         int hourly, daily;
         incr_rcpt_count(priv.mailaddr, hourly, daily);
-        int h_limit = dc.default_context->find_rate(priv.mailaddr);
-        int d_limit = dc.default_context->get_daily_multiple() * h_limit;
+        int h_limit = dc.default_context->find_rate_limit(priv.mailaddr);
+        int d_limit = dc.default_context->get_daily_rate_multiple() * h_limit;
         if (debug_syslog > 1) {
             char msg[maxlen];
             snprintf(msg, sizeof(msg), "unauthenticated address %s (%d %d recipients, %d %d limits)", priv.mailaddr, hourly, daily, h_limit, d_limit);
@@ -1600,9 +1651,12 @@
             // three minutes thru each loop, 20 loops per hour
             // clear the recipient hourly counts
             pthread_mutex_lock(&rate_mutex);
-                for (rcpt_rates::iterator i=rcpt_hourly_counts.begin(); i!=rcpt_hourly_counts.end(); i++) {
+                for (rates::iterator i=rcpt_hourly_counts.begin(); i!=rcpt_hourly_counts.end(); i++) {
                     (*i).second = 0;
                 }
+                for (auth_addresses::iterator j=auth_hourly_addresses.begin(); j!=auth_hourly_addresses.end(); j++) {
+                    delete (*j).second;
+                }
             pthread_mutex_unlock(&rate_mutex);
             loop1 = 0;
         }
@@ -1610,9 +1664,12 @@
             // three minutes thru each loop, 480 loops per day
             // clear the recipient daily counts
             pthread_mutex_lock(&rate_mutex);
-                for (rcpt_rates::iterator i=rcpt_daily_counts.begin(); i!=rcpt_daily_counts.end(); i++) {
+                for (rates::iterator i=rcpt_daily_counts.begin(); i!=rcpt_daily_counts.end(); i++) {
                     (*i).second = 0;
                 }
+                for (auth_addresses::iterator j=auth_daily_addresses.begin(); j!=auth_daily_addresses.end(); j++) {
+                    delete (*j).second;
+                }
             pthread_mutex_unlock(&rate_mutex);
             loop2 = 0;
         }
--- a/tld.conf	Wed Dec 11 22:57:06 2013 -0800
+++ b/tld.conf	Tue Dec 17 15:35:23 2013 -0800
@@ -1,4 +1,4 @@
-// generated by make-tld-conf.py on 2013-12-11 22:55:22
+// generated by make-tld-conf.py on 2013-12-12 08:31:06
 
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -6380,7 +6380,7 @@
 k12.va.us
 k12.wa.us
 k12.wi.us
-k12.wv.us
+// k12.wv.us  Bug 947705 - Removed at request of Verne Britton <verne@wvnet.edu>
 k12.wy.us
 
 cc.ak.us
--- a/xml/dnsbl.in	Wed Dec 11 22:57:06 2013 -0800
+++ b/xml/dnsbl.in	Tue Dec 17 15:35:23 2013 -0800
@@ -612,6 +612,10 @@
                 http:// protocol header. Such references are still clickable in common
                 mail software.
             </para>
+            <para>
+                Add trusted_spf list of domains for which we trust their spf data. Mail
+                from their listed ip addresses from their domain will be whitelisted.
+            </para>
         </refsect1>
 
         <refsect1 id='copyright.1'>
@@ -734,11 +738,14 @@
 FROM-ADDR  = ADDRESS VALUE [";"]
 DCC-FROM   = "dcc_from" "{" DCCINCLUDEFILE "}" ";"
 
-RATE-LIMIT     = "rate_limit" [DEFAULT_LIMIT [DAILY_MULTIPLE]] "{" (RATE)+ "}"
-RATE           = USER LIMIT [";"]
-LIMIT          = INTEGER
-DEFAULT_LIMIT  = INTEGER
-DAILY_MULTIPLE = INTEGER
+RATE-LIMIT     = "rate_limit" DEFAULT_RCPT_LIMIT DAILY_MULTIPLE_RCPT
+                              DEFAULT_IP_LIMIT   DAILY_MULTIPLE_IP "{" (RATE)+ "}"
+RATE           = USER RCPTLIMIT IPLIMIT ";"
+RCPTLIMIT      = INTEGER
+DEFAULT_RCPT_LIMIT  = INTEGER
+DAILY_MULTIPLE_RCPT = INTEGER
+DEFAULT_IP_LIMIT    = INTEGER
+DAILY_MULTIPLE_IP   = INTEGER
 
 DEFAULT    = ("white" | "black" | "unknown" | "inherit" | "")
 ADDRESS    = (USER@ | DOMAIN | USER@DOMAIN)
@@ -772,21 +779,23 @@
         dcc_bulk_threshold  50;
     };
 
-    // backscatter prevention - don't send bounces for mail that we accepted but could not forward
+    // backscatter prevention - do not send bounces for mail that we accepted but could not forward
     // we only send bounces to our own customers
     env_from unknown {
         "<>"    black;
     };
 
-    // hourly recipient rate limit by smtp auth client id,
-    // or unauthenticated mail from address
-    // default hourly limit is 30
-    // daily limits are 4 times the hourly limit
-    rate_limit 30 4 { // default
-        #fred 100;   // override default limits
-        #joe  10;    // ""
-        #"sam@somedomain.tld"  500;
-        #"@otherdomain.tld"    100;
+    // hourly recipient rate limit by smtp auth client id, or unauthenticated mail from address
+    // hourly unique ip addresses  by smtp auth client id, or unauthenticated mail from address
+    // default hourly rate limit is 30
+    // daily rate limits are 4 times the hourly limit
+    // default hourly unique ip addresses is 5
+    // daily unique ip addresses are 4 times the hourly limit
+    rate_limit 30 4 5 4 { // default
+        fred 100 10;   // override default limits
+        joe  10  2;    // ""
+        "sam@somedomain.tld"  500 2;
+        "@otherdomain.tld"    100 2;
     };
 };