changeset 382:c378e9d03f37

start parsing spf txt records
author Carl Byington <carl@five-ten-sg.com>
date Mon, 06 Mar 2017 12:21:05 -0800
parents 879a470c6ac3
children 5a88320a7547
files ChangeLog dnsbl.spec.in src/context.cpp src/context.h src/dnsbl.cpp src/dnsbl.h src/includes.h
diffstat 7 files changed, 137 insertions(+), 77 deletions(-) [+]
line wrap: on
line diff
--- a/ChangeLog	Tue Feb 28 17:02:07 2017 -0800
+++ b/ChangeLog	Mon Mar 06 12:21:05 2017 -0800
@@ -1,5 +1,5 @@
-6.51 2017-02-28
-    fetch spf txt records for required dkim signers
+6.51 2017-03-06
+    parse spf txt records for required dkim signers
 
 6.50 2017-02-22
     reject if dkim signer is listed on surbl
--- a/dnsbl.spec.in	Tue Feb 28 17:02:07 2017 -0800
+++ b/dnsbl.spec.in	Mon Mar 06 12:21:05 2017 -0800
@@ -3,7 +3,7 @@
 Summary:            Sendmail milter for spam control
 Name:               @PACKAGE@
 Version:            @VERSION@
-Release:            1%{?dist}
+Release:            2%{?dist}
 License:            GPLv3+
 Group:              System Environment/Daemons
 Source:             http://www.five-ten-sg.com/%{name}/packages/%{name}-%{version}.tar.gz
@@ -155,6 +155,9 @@
 
 
 %changelog
+* Mon Mar 06 2017 Carl Byington <carl@five-ten-sg.com> - 6.51-2
+- parse spf txt records.
+
 * Tue Feb 28 2017 Carl Byington <carl@five-ten-sg.com> - 6.51-1
 - fetch spf txt records for required dkim signers.
 
--- a/src/context.cpp	Tue Feb 28 17:02:07 2017 -0800
+++ b/src/context.cpp	Mon Mar 06 12:21:05 2017 -0800
@@ -9,6 +9,7 @@
 #include "includes.h"
 
 #include <arpa/inet.h>
+#include <arpa/nameser.h>
 #include <net/if.h>
 #include <netdb.h>
 #include <netinet/in.h>
@@ -1122,17 +1123,52 @@
 }
 
 
-#ifdef NS_PACKETSZ
-bool CONTEXT::resolve_spf(const char *from, int32_t ip, mlfiPriv *priv)
+bool CONTEXT::resolve_spf(const char *from, uint32_t ip, mlfiPriv *priv, int level)
 {
     char buf[maxlen];
     dns_interface(*priv, from, ns_t_txt, false, NULL, buf, maxlen);
     if (*buf) {
-        log(priv->queueid, "found txt record %s", buf);
+        log(priv->queueid, "found txt record for %s", from);
+        log(priv->queueid, "found txt record value %s", buf);
+        char *p = buf;
+        char *e = p + strlen(p);    // point to trailing null
+        while (p = strstr(p, " ip4:")) {
+            p += 5;
+            char *b = strchr(p, ' ');
+            if (b) *b = '\0';
+            char *s = strchr(p, '/');
+            if (s) *s = '\0';
+            in_addr ipx;
+            if (inet_aton(p, &ipx)) {
+                if (s) {
+                    int mask = atoi(s+1);
+                    if ((mask >= 16) && (mask <= 32)) {
+                        int low = (1 << (32-mask)) - 1;
+                        ipx.s_addr &= low ^ 0xffffffff;
+                        if ((ip >= ipx.s_addr) && (ip <= ipx.s_addr + low)) {
+                            log(priv->queueid, "match %s", p);
+                            if (s) log(priv->queueid, "match /%s", s+1);
+                            return true;
+                        }
+                    }
+                }
+            }
+            if (b) *b = ' ';
+            if (s) *s = '/';
+            p = (b) ? b+1 : e;
+        }
+        p = buf;
+        while ((level < 5) && (p = strstr(p, " include:"))) {
+            p += 9;
+            char *b = strchr(p, ' ');
+            if (b) *b = '\0';
+            if (resolve_spf(p, ip, priv, level+1)) return true;
+            if (b) *b = ' ';
+            p = (b) ? b+1 : e;
+        }
     }
     return false;
 }
-#endif
 
 
 const char *CONTEXT::acceptable_content(recorder &memory, int score, int bulk, const char *queueid, string_set &signers, const char *from, mlfiPriv *priv, string& msg) {
@@ -1149,6 +1185,12 @@
 
     if (dk) {
         const char *st = dk->action;
+        if ((st == token_require_signed) &&
+            dk->signer &&
+            strcmp(dk->signer, " ") &&
+            resolve_spf(from, priv->ip, priv)) {
+                log(queueid, "spf pass for %s with required dkim signer", from);
+        }
         for (string_set::iterator s=signers.begin(); s!=signers.end(); s++) {
             // signed by a white listed signer
             if ((st == token_signed_white)   && in_signing_set(*s,dk->signer)) {
@@ -1169,13 +1211,12 @@
             }
         }
         if (st == token_require_signed) {
-#ifdef NS_PACKETSZ
-            // not signed by the required signers, but maybe passes strong spf check
-            if (resolve_spf(from, priv->ip, priv) {
+            // not signed by a required signer, but maybe passes strong spf check
+            // only check spf if the list of required signers is not a single blank.
+            if (dk->signer && strcmp(dk->signer, " ") && resolve_spf(from, priv->ip, priv)) {
                 log(queueid, "spf pass for %s rather than required dkim signer", from);
                 return token_white;
             }
-#endif
             char buf[maxlen];
             snprintf(buf, sizeof(buf), "Mail rejected - not dkim signed by %s", dk->signer);
             msg = string(buf);
--- a/src/context.h	Tue Feb 28 17:02:07 2017 -0800
+++ b/src/context.h	Mon Mar 06 12:21:05 2017 -0800
@@ -29,6 +29,8 @@
 typedef set<int>                                int_set;
 typedef set<int32_t>                            int32_t_set;
 typedef int32_t_set *                           int32_t_set_p;
+typedef set<uint32_t>                               uint32_t_set;
+typedef uint32_t_set *                              uint32_t_set_p;
 typedef list<SMTP *>                            smtp_list;
 typedef DKIM *                                  DKIMP;
 typedef DNSBL *                                 DNSBLP;
@@ -44,9 +46,9 @@
 typedef CONTEXT *                               CONTEXTP;
 typedef list<CONTEXTP>                          context_list;
 typedef map<const char *, CONTEXTP, ltstr>      context_map;
-typedef map<const char *, int32_t, ltstr>       ns_mapper;  // name to ipv4 address
+typedef map<const char *, uint32_t, ltstr>          ns_mapper;  // name to ipv4 address
 typedef map<const char *, int, ltstr>           rates;
-typedef map<const char *, int32_t_set_p, ltstr> auth_addresses;
+typedef map<const char *, uint32_t_set_p, ltstr>    auth_addresses;
 typedef map<const char *, time_t,  ltstr>       autowhite_sent;
 typedef map<const char *, VERIFYP, ltstr>       verify_map;
 typedef map<const char *, WHITELISTERP, ltstr>  whitelister_map;
@@ -315,6 +317,7 @@
 
     void        log(const char *queueid, const char *msg, const char *v);
     bool        in_signing_set(const char *s, const char *signers);
+    bool        resolve_spf(const char *from, uint32_t ip, mlfiPriv *priv, int level = 0);
     const char *acceptable_content(recorder &memory, int score, int bulk, const char *queueid, string_set &signers, const char *from, mlfiPriv *priv, string& msg);
     bool        ignore_host(const char *host);
 
--- a/src/dnsbl.cpp	Tue Feb 28 17:02:07 2017 -0800
+++ b/src/dnsbl.cpp	Mon Mar 06 12:21:05 2017 -0800
@@ -123,14 +123,6 @@
 auth_addresses auth_daily_addresses;        // protected with rate_mutex
 
 
-struct ns_map {
-    // all the strings are owned by the keys/values in the ns_host string map
-    string_map  ns_host;    // nameserver name -> host name that uses this name server
-    ns_mapper   ns_ip;      // nameserver name -> ipv4 address of the name server
-    ~ns_map();
-    void add(const char *name, const char *refer);
-};
-
 
 ns_map::~ns_map() {
     for (string_map::iterator i=ns_host.begin(); i!=ns_host.end(); i++) {
@@ -195,18 +187,18 @@
 }
 
 
-void add_auth_address(const char *user, int &hourly, int &daily, int32_t ip);
-void add_auth_address(const char *user, int &hourly, int &daily, int32_t ip) {
+void add_auth_address(const char *user, int &hourly, int &daily, uint32_t ip);
+void add_auth_address(const char *user, int &hourly, int &daily, uint32_t ip) {
     pthread_mutex_lock(&rate_mutex);
         auth_addresses::iterator i = auth_hourly_addresses.find(user);
         if (i == auth_hourly_addresses.end()) {
             user = strdup(user);
-            auth_hourly_addresses[user] = new int32_t_set;
+            auth_hourly_addresses[user] = new uint32_t_set;
             auth_hourly_addresses[user]->insert(ip);
             hourly = 1;
         }
         else {
-            int32_t_set::iterator k = ((*i).second)->find(ip);
+            uint32_t_set::iterator k = ((*i).second)->find(ip);
             if (k == ((*i).second)->end()) ((*i).second)->insert(ip);
             hourly = ((*i).second)->size();
         }
@@ -214,12 +206,12 @@
         auth_addresses::iterator j = auth_daily_addresses.find(user);
         if (j == auth_daily_addresses.end()) {
             user = strdup(user);
-            auth_daily_addresses[user] = new int32_t_set;
+            auth_daily_addresses[user] = new uint32_t_set;
             auth_daily_addresses[user]->insert(ip);
             daily = 1;
         }
         else {
-            int32_t_set::iterator k = ((*j).second)->find(ip);
+            uint32_t_set::iterator k = ((*j).second)->find(ip);
             if (k == ((*j).second)->end()) ((*j).second)->insert(ip);
             daily = ((*j).second)->size();
         }
@@ -307,10 +299,13 @@
 //  If we cannot get an answer, we just accept the mail.
 //  If the qtype is ns_t_txt, the answer is placed in txt_answer which
 //  must be non-null, and the return value can be ignored.
+//  A null string is returned in txt_answer in the case of errors.
 //  If the qtype is ns_t_a, the ip address is returned in network byte order.
+//  IP address 0 is returned in case of errors.
 //
-uint32_t dns_interface(mlfiPriv &priv, const char *question, int qtype, bool maybe_ip = false, ns_map *nameservers = NULL, char *txt_answer = NULL, size_t txt_size = 0);
 uint32_t dns_interface(mlfiPriv &priv, const char *question, int qtype, bool maybe_ip, ns_map *nameservers, char *txt_answer, size_t txt_size) {
+    if (txt_answer) txt_answer[0] = '\0';   // return null string if there are no txt answers
+
     // tell sendmail we are still working
     #if _FFR_SMFI_PROGRESS
         if (priv.eom) smfi_progress(priv.ctx);
@@ -419,9 +414,9 @@
                     }
                 }
             }
-            if ((qtype == ns_t_txt) && (txt_answer) && (txt_size > 5)) {
+            if ((qtype == ns_t_txt) && (txt_answer) && (txt_size > 7)) {
+                txt_answer[0] = '\0';  // return null string if there are no txt answers
                 txt_size--; // allow room for terminating null;
-                txt_answer[0] = '\0';   // return null string if there are no answers
                 while (ns_parserr(&handle, ns_s_an, rrnum++, &rr) == 0) {
                     size_t offset = 0;
                     if (ns_rr_type(rr) == qtype) {
@@ -439,7 +434,10 @@
                         }
                     }
                     txt_answer[offset] = '\0';  // trailing null
-                    if (strcasecmp(txt_answer, "v=spf1 ") == 0) break;
+                    if (strncasecmp(txt_answer, "v=spf1 ", 7) == 0) break;
+                }
+                if (strncasecmp(txt_answer, "v=spf1 ", 7) != 0) {
+                    txt_answer[0] = '\0';  // return null string if there are no spf1 txt answers
                 }
             }
         }
@@ -891,10 +889,16 @@
 #else
         glom.length = sizeof(glom.answer);
         glom.answer = 0;
+        int t = int8_t(question[0]);
+        if (t != ns_t_a) {
+            glom.length = 0;
+        }
+        else {
         struct hostent *host = gethostbyname(question+1);
         if (host && (host->h_addrtype == AF_INET)) {
             memcpy(&glom.answer, host->h_addr, sizeof(glom.answer));
         }
+        }
 #endif
 
         // write the answer
@@ -927,8 +931,8 @@
 ////////////////////////////////////////////////
 //  check a single dns list, return ip address in network byte order
 //
-uint32_t check_single(mlfiPriv &priv, int32_t ip, const char *suffix);
-uint32_t check_single(mlfiPriv &priv, int32_t ip, const char *suffix) {
+uint32_t check_single(mlfiPriv &priv, uint32_t ip, const char *suffix);
+uint32_t check_single(mlfiPriv &priv, uint32_t ip, const char *suffix) {
     // make a dns question
     const u_char *src = (const u_char *)&ip;
     if (src[0] == 127) return 0;    // don't do dns lookups on localhost
@@ -949,8 +953,8 @@
 ////////////////////////////////////////////////
 //  check a single dnsbl
 //
-bool check_single(mlfiPriv &priv, int32_t ip, DNSBL &bl);
-bool check_single(mlfiPriv &priv, int32_t ip, DNSBL &bl) {
+bool check_single(mlfiPriv &priv, uint32_t ip, DNSBL &bl);
+bool check_single(mlfiPriv &priv, uint32_t ip, DNSBL &bl) {
     return check_single(priv, ip, bl.suffix);
 }
 
@@ -958,8 +962,8 @@
 ////////////////////////////////////////////////
 //  check a single dnswl
 //
-bool check_single(mlfiPriv &priv, int32_t ip, DNSWL &wl);
-bool check_single(mlfiPriv &priv, int32_t ip, DNSWL &wl) {
+bool check_single(mlfiPriv &priv, uint32_t ip, DNSWL &wl);
+bool check_single(mlfiPriv &priv, uint32_t ip, DNSWL &wl) {
     uint32_t r = ntohl(check_single(priv, ip, wl.suffix));
     uint32_t v = (uint32_t)0x7f000000;
     uint32_t m = (uint32_t)0xffff0000;
@@ -1026,8 +1030,8 @@
 //  check the hosts from the body against the content filter and uribl dnsbls
 //
 //
-bool check_hosts(mlfiPriv &priv, bool random, int limit, const char *&msg, const char *&host, int32_t &ip, const char *&found);
-bool check_hosts(mlfiPriv &priv, bool random, int limit, const char *&msg, const char *&host, int32_t &ip, const char *&found) {
+bool check_hosts(mlfiPriv &priv, bool random, int limit, const char *&msg, const char *&host, uint32_t &ip, const char *&found);
+bool check_hosts(mlfiPriv &priv, bool random, int limit, const char *&msg, const char *&host, uint32_t &ip, const char *&found) {
     found = NULL;   // normally ip address style
     if (!priv.content_suffix && !priv.uribl_suffix) return false;   // nothing to check
     string_set &hosts  = priv.memory->get_hosts();
@@ -1035,7 +1039,7 @@
 
     int count = 0;
     int   cnt = hosts.size();   // number of hosts we could look at
-    int32_t_set ips;
+    uint32_t_set ips;
     ns_map  nameservers;
     for (string_set::iterator i=hosts.begin(); i!=hosts.end(); i++) {
         host = *i;  // a reference into hosts, which will live until this smtp transaction is closed
@@ -1072,7 +1076,7 @@
             my_syslog(&priv, buf);
         }
         if (ip) {
-            int32_t_set::iterator i = ips.find(ip);
+            uint32_t_set::iterator i = ips.find(ip);
             if (i == ips.end()) {
                 // we haven't looked this up yet
                 ips.insert(ip);
@@ -1110,7 +1114,7 @@
             my_syslog(&priv, buf);
         }
         if (ip) {
-            int32_t_set::iterator i = ips.find(ip);
+            uint32_t_set::iterator i = ips.find(ip);
             if (i == ips.end()) {
                 ips.insert(ip);
                 if (check_single(priv, ip, priv.content_suffix)) {
@@ -1649,7 +1653,7 @@
     sfsistat  rc;
     mlfiPriv   &priv = *MLFIPRIV;
     const char *host = NULL;
-    int32_t     ip;
+    uint32_t    ip;
     // process end of message
     priv.eom = true;
     if (priv.authenticated || priv.only_whites) rc = SMFIS_CONTINUE;
@@ -1847,7 +1851,7 @@
                 }
                 for (auth_addresses::iterator j=auth_hourly_addresses.begin(); j!=auth_hourly_addresses.end(); j++) {
                     delete (*j).second;
-                    (*j).second = new int32_t_set;
+                    (*j).second = new uint32_t_set;
                 }
             pthread_mutex_unlock(&rate_mutex);
             loop1 = 0;
@@ -1861,7 +1865,7 @@
                 }
                 for (auth_addresses::iterator j=auth_daily_addresses.begin(); j!=auth_daily_addresses.end(); j++) {
                     delete (*j).second;
-                    (*j).second = new int32_t_set;
+                    (*j).second = new uint32_t_set;
                 }
             pthread_mutex_unlock(&rate_mutex);
             loop2 = 0;
--- a/src/dnsbl.h	Tue Feb 28 17:02:07 2017 -0800
+++ b/src/dnsbl.h	Mon Mar 06 12:21:05 2017 -0800
@@ -12,7 +12,6 @@
 #include "context.h"
 #include "spamass.h"
 #include "dccifd.h"
-#include <stdint.h>
 
 extern int debug_syslog;
 #define dccbulk 1000
@@ -85,9 +84,18 @@
     void need_content_filter(CONTEXT &con);
 };
 
+struct ns_map {
+    // all the strings are owned by the keys/values in the ns_host string map
+    string_map  ns_host;    // nameserver name -> host name that uses this name server
+    ns_mapper   ns_ip;      // nameserver name -> ipv4 address of the name server
+    ~ns_map();
+    void add(const char *name, const char *refer);
+};
+
 void my_syslog(const char *queueid, const char *text);
 void my_syslog(mlfiPriv *priv, const char *text);
 void my_syslog(mlfiPriv *priv, const string text);
 void my_syslog(const char *text);
+uint32_t dns_interface(mlfiPriv &priv, const char *question, int qtype, bool maybe_ip = false, ns_map *nameservers = NULL, char *txt_answer = NULL, size_t txt_size = 0);
 
 #endif
--- a/src/includes.h	Tue Feb 28 17:02:07 2017 -0800
+++ b/src/includes.h	Mon Mar 06 12:21:05 2017 -0800
@@ -24,6 +24,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string>
+#include <stdint.h>
 
 #include "tokenizer.h"
 #include "context.h"