--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/new.cpp	Sat Jul 09 19:24:41 2005 -0700
@@ -0,0 +1,1389 @@
+/*
+
+Copyright (c) 2004, 2005 Carl Byington - 510 Software Group, released
+under the GPL version 2 or any later version at your choice available at
+http://www.fsf.org/licenses/gpl.txt
+
+Based on a sample milter Copyright (c) 2000-2003 Sendmail, Inc. and its
+suppliers.  Inspired by the DCC by Rhyolite Software
+
+-r port  The port used to talk to our internal dns resolver processes
+-p port  The port through which the MTA will connect to this milter.
+-t sec   The timeout value.
+-c       Check the config, and print a copy to stdout. Don't start the
+         milter or do anything with the socket.
+-d       Add debug syslog entries
+
+
+TODO:
+1) Add config for max_recipients for each mail domain. Recipients in
+excess of that limit will be rejected, and the entire data will be
+rejected if it is sent.
+
+2) Add config for poison addresses. If any recipient is poison, all
+recipients are rejected even if they would be whitelisted, and the
+data is rejected if sent.
+
+3) Add option to only allow one recipient if the return path is empty.
+
+4) Check if the envelope from domain name primary MX points 127.0.0.0/8
+
+5) Add option for using smtp connections to verify addresses from backup
+mx machines. This allows the backup mx to learn the valid addresses
+on the primary machine.
+
+*/
+
+
+// from sendmail sample
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <sysexits.h>
+#include <unistd.h>
+
+// needed for socket io
+#include <sys/ioctl.h>
+#include <net/if.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+// needed for thread
+#include <pthread.h>
+
+// needed for std c++ collections
+#include <set>
+#include <map>
+#include <list>
+
+// for the dns resolver
+#include <netinet/in.h>
+#include <arpa/nameser.h>
+#include <resolv.h>
+
+// misc stuff needed here
+#include <ctype.h>
+#include <syslog.h>
+#include <pwd.h>
+#include <sys/wait.h>   /* header for waitpid() and various macros */
+#include <signal.h>     /* header for signal functions */
+
+#include "context.h"
+
+static char* dnsbl_version="$Id:";
+
+extern "C" {
+    #include "libmilter/mfapi.h"
+    sfsistat mlfi_connect(SMFICTX *ctx, char *hostname, _SOCK_ADDR *hostaddr);
+    sfsistat mlfi_envfrom(SMFICTX *ctx, char **argv);
+    sfsistat mlfi_envrcpt(SMFICTX *ctx, char **argv);
+    sfsistat mlfi_body(SMFICTX *ctx, u_char *data, size_t len);
+    sfsistat mlfi_eom(SMFICTX *ctx);
+    sfsistat mlfi_abort(SMFICTX *ctx);
+    sfsistat mlfi_close(SMFICTX *ctx);
+    void sig_chld(int signo);
+}
+
+struct ns_map {
+    // all the strings are owned by the keys/values in the ns_host string map
+    string_map  ns_host;    // nameserver name -> host name that uses this name server
+    ns_mapper   ns_ip;      // nameserver name -> ip address of the name server
+};
+
+static bool debug_syslog = false;
+static bool loader_run   = true;    // used to stop the config loader thread
+static CONFIG * config = NULL;      // protected by the config_mutex
+static int  generation = 0;         // protected by the config_mutex
+
+static pthread_mutex_t  config_mutex;
+static pthread_mutex_t  syslog_mutex;
+static pthread_mutex_t  resolve_mutex;
+static pthread_mutex_t  fd_pool_mutex;
+
+static std::set<int>    fd_pool;
+static int    NULL_SOCKET       = -1;
+static char  *resolver_port     = NULL;         // unix domain socket to talk to the dns resolver process
+static int    resolver_socket   = NULL_SOCKET;  // socket used to listen for resolver requests
+static time_t ERROR_SOCKET_TIME = 60;           // number of seconds between attempts to open the spam filter socket
+static time_t last_error_time;
+static int    resolver_sock_count = 0;          // protected with fd_pool_mutex
+static int    resolver_pool_size  = 0;          // protected with fd_pool_mutex
+
+
+// packed structure to allow a single socket write to dump the
+// length and the following answer. The packing attribute is gcc specific.
+struct glommer {
+    int    length;
+    #ifdef NS_PACKETSZ
+        u_char answer[NS_PACKETSZ];     // with a resolver, we return resolver answers
+    #else
+        int    answer;                  // without a resolver, we return a single ip4 address, 0 == no answer
+    #endif
+} __attribute__ ((packed));
+
+struct mlfiPriv;
+
+
+////////////////////////////////////////////////
+// helper to discard the strings and objects held by an ns_map
+//
+static void discard(ns_map &s);
+static void discard(ns_map &s) {
+    for (string_map::iterator i=s.ns_host.begin(); i!=s.ns_host.end(); i++) {
+        char *x = (*i).first;
+        char *y = (*i).second;
+        free(x);
+        free(y);
+    }
+    s.ns_ip.clear();
+    s.ns_host.clear();
+}
+
+////////////////////////////////////////////////
+// helper to register a string in an ns_map
+//
+static void register_string(ns_map &s, char *name, char *refer);
+static void register_string(ns_map &s, char *name, char *refer) {
+    string_map::iterator i = s.ns_host.find(name);
+    if (i != s.ns_host.end()) return;
+    char *x = strdup(name);
+    char *y = strdup(refer);
+    s.ns_ip[x]   = 0;
+    s.ns_host[x] = y;
+
+}
+
+////////////////////////////////////////////////
+// syslog a message
+//
+static void my_syslog(mlfiPriv *priv, char *text);
+
+
+// include the content scanner
+#include "scanner.cpp"
+
+
+////////////////////////////////////////////////
+// disconnect the fd from the dns resolver process
+//
+void my_disconnect(int sock, bool decrement = true);
+void my_disconnect(int sock, bool decrement)
+{
+    if (sock != NULL_SOCKET) {
+        if (decrement) {
+            pthread_mutex_lock(&fd_pool_mutex);
+                resolver_sock_count--;
+            pthread_mutex_unlock(&fd_pool_mutex);
+        }
+        shutdown(sock, SHUT_RDWR);
+        close(sock);
+    }
+}
+
+
+////////////////////////////////////////////////
+// return fd connected to the dns resolver process
+//
+int my_connect();
+int my_connect()
+{
+    // if we have had recent errors, don't even try to open the socket
+    time_t now = time(NULL);
+    if ((now - last_error_time) < ERROR_SOCKET_TIME) return NULL_SOCKET;
+
+    // nothing recent, maybe this time it will work
+    int sock = NULL_SOCKET;
+    sockaddr_un server;
+    memset(&server, '\0', sizeof(server));
+    server.sun_family = AF_UNIX;
+    strncpy(server.sun_path, resolver_port, sizeof(server.sun_path)-1);
+    sock = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (sock != NULL_SOCKET) {
+        bool rc = (connect(sock, (sockaddr *)&server, sizeof(server)) == 0);
+        if (!rc) {
+            my_disconnect(sock, false);
+            sock = NULL_SOCKET;
+            last_error_time = now;
+        }
+    }
+    else last_error_time = now;
+    if (sock != NULL_SOCKET) {
+        pthread_mutex_lock(&fd_pool_mutex);
+            resolver_sock_count++;
+        pthread_mutex_unlock(&fd_pool_mutex);
+    }
+    return sock;
+}
+
+
+////////////////////////////////////////////////
+// mail filter private data, held for us by sendmail
+//
+struct mlfiPriv
+{
+    // connection specific data
+    CONFIG  *pc;                    // global context with our maps
+    int     fd;                     // to talk to dns resolvers process
+    bool    err;                    // did we get any errors on the resolver socket?
+    int     ip;                     // ip4 address of the smtp client
+    map<DNSBLP, status> checked;    // status from those lists
+    // message specific data
+    char    *mailaddr;      // envelope from value
+    char    *queueid;       // sendmail queue id
+    bool    authenticated;  // client authenticated? if so, suppress all dnsbl checks
+    bool    have_whites;    // have at least one whitelisted recipient? need to accept content and remove all non-whitelisted recipients if it fails
+    bool    only_whites;    // every recipient is whitelisted?
+    string_set  non_whites; // remember the non-whitelisted recipients so we can remove them if need be
+    recorder    *memory;    // memory for the content scanner
+    url_scanner *scanner;   // object to handle body scanning
+    mlfiPriv();
+    ~mlfiPriv();
+    void reset(bool final = false); // for a new message
+    void get_fd();
+    void return_fd();
+    int  my_read(char *buf, int len);
+    int  my_write(char *buf, int len);
+};
+
+mlfiPriv::mlfiPriv() {
+    pthread_mutex_lock(&config_mutex);
+        pc = config;
+        pc->reference_count++;
+    pthread_mutex_unlock(&config_mutex);
+    get_fd();
+    ip            = 0;
+    mailaddr      = NULL;
+    queueid       = NULL;
+    authenticated = false;
+    have_whites   = false;
+    only_whites   = true;
+    memory        = new recorder(this, &pc->html_tags, &pc->tlds);
+    scanner       = new url_scanner(memory);
+}
+
+mlfiPriv::~mlfiPriv() {
+    return_fd();
+    pthread_mutex_lock(&config_mutex);
+        pc->reference_count--;
+    pthread_mutex_unlock(&config_mutex);
+    reset(true);
+}
+
+void mlfiPriv::reset(bool final) {
+    if (mailaddr) free(mailaddr);
+    if (queueid)  free(queueid);
+    discard(non_whites);
+    delete memory;
+    delete scanner;
+    if (!final) {
+        mailaddr      = NULL;
+        queueid       = NULL;
+        authenticated = false;
+        have_whites   = false;
+        only_whites   = true;
+        memory        = new recorder(this, &pc->html_tags, &pc->tlds);
+        scanner       = new url_scanner(memory);
+    }
+}
+
+void mlfiPriv::get_fd()
+{
+    err = true;
+    fd  = NULL_SOCKET;
+    int result = pthread_mutex_lock(&fd_pool_mutex);
+    if (!result) {
+        std::set<int>::iterator i;
+        i = fd_pool.begin();
+        if (i != fd_pool.end()) {
+            // have at least one fd in the pool
+            err = false;
+            fd  = *i;
+            fd_pool.erase(fd);
+            resolver_pool_size--;
+            pthread_mutex_unlock(&fd_pool_mutex);
+        }
+        else {
+            // pool is empty, get a new fd
+            pthread_mutex_unlock(&fd_pool_mutex);
+            fd  = my_connect();
+            err = (fd == NULL_SOCKET);
+        }
+    }
+    else {
+        // cannot lock the pool, just get a new fd
+        fd  = my_connect();
+        err = (fd == NULL_SOCKET);
+    }
+}
+
+void mlfiPriv::return_fd()
+{
+    if (err) {
+        // this fd got a socket error, so close it, rather than returning it to the pool
+        my_disconnect(fd);
+    }
+    else {
+        int result = pthread_mutex_lock(&fd_pool_mutex);
+        if (!result) {
+            if ((resolver_sock_count > resolver_pool_size*5) || (resolver_pool_size < 5)) {
+                // return the fd to the pool
+                fd_pool.insert(fd);
+                resolver_pool_size++;
+                pthread_mutex_unlock(&fd_pool_mutex);
+            }
+            else {
+                // more than 20% of the open resolver sockets are in the pool, and the
+                // pool as at least 5 sockets. that is enough, so just close this one.
+                pthread_mutex_unlock(&fd_pool_mutex);
+                my_disconnect(fd);
+            }
+        }
+        else {
+            // could not lock the pool, so just close the fd
+            my_disconnect(fd);
+        }
+    }
+}
+
+int mlfiPriv::my_write(char *buf, int len)
+{
+    if (err) return 0;
+    int rs = 0;
+    while (len) {
+        int ws = write(fd, buf, len);
+        if (ws > 0) {
+            rs  += ws;
+            len -= ws;
+            buf += ws;
+        }
+        else {
+            // peer closed the socket!
+            rs = 0;
+            err = true;
+            break;
+        }
+    }
+    return rs;
+}
+
+int mlfiPriv::my_read(char *buf, int len)
+{
+    if (err) return 0;
+    int rs = 0;
+    while (len > 1) {
+        int ws = read(fd, buf, len);
+        if (ws > 0) {
+            rs  += ws;
+            len -= ws;
+            buf += ws;
+        }
+        else {
+            // peer closed the socket!
+            rs = 0;
+            err = true;
+            break;
+        }
+    }
+    return rs;
+}
+
+#define MLFIPRIV    ((struct mlfiPriv *) smfi_getpriv(ctx))
+
+
+////////////////////////////////////////////////
+// syslog a message
+//
+static void my_syslog(mlfiPriv *priv, char *text) {
+    char buf[1000];
+    if (priv) {
+        snprintf(buf, sizeof(buf), "%s: %s", priv->queueid, text);
+        text = buf;
+    }
+    pthread_mutex_lock(&syslog_mutex);
+        openlog("dnsbl", LOG_PID, LOG_MAIL);
+        syslog(LOG_NOTICE, "%s", text);
+        closelog();
+    pthread_mutex_unlock(&syslog_mutex);
+}
+
+static void my_syslog(char *text);
+static void my_syslog(char *text) {
+    my_syslog(NULL, text);
+}
+
+////////////////////////////////////////////////
+// register a global string
+//
+static char* register_string(char *name);
+static char* register_string(char *name) {
+    return register_string(all_strings, name);
+}
+
+
+static char* next_token(char *delim);
+static char* next_token(char *delim) {
+    char *name = strtok(NULL, delim);
+    if (!name) return name;
+    return register_string(name);
+}
+
+
+////////////////////////////////////////////////
+// lookup an email address in the env_from or env_to maps
+//
+static char* lookup1(char *email, string_map map);
+static char* lookup1(char *email, string_map map) {
+    string_map::iterator i = map.find(email);
+    if (i != map.end()) return (*i).second;
+    char *x = strchr(email, '@');
+    if (!x) return DEFAULT;
+    x++;
+    i = map.find(x);
+    if (i != map.end()) return (*i).second;
+    return DEFAULT;
+}
+
+
+////////////////////////////////////////////////
+// lookup an email address in the env_from or env_to maps
+// this email address is passed in from sendmail, and will
+// always be enclosed in <>. It may have mixed case, just
+// as the mail client sent it.
+//
+static char* lookup(char* email, string_map map);
+static char* lookup(char* email, string_map map) {
+    int n = strlen(email)-2;
+    if (n < 1) return DEFAULT;  // malformed
+    char *key = strdup(email+1);
+    key[n] = '\0';
+    for (int i=0; i<n; i++) key[i] = tolower(key[i]);
+    char *rc = lookup1(key, map);
+    free(key);
+    return rc;
+}
+
+
+////////////////////////////////////////////////
+//  find the dnsbl with a specific name
+//
+static DNSBLP find_dnsbl(CONFIG &dc, char *name);
+static DNSBLP find_dnsbl(CONFIG &dc, char *name) {
+    dnsblp_map::iterator i = dc.dnsbls.find(name);
+    if (i == dc.dnsbls.end()) return NULL;
+    return (*i).second;
+}
+
+
+////////////////////////////////////////////////
+//  find the dnsbll with a specific name
+//
+static DNSBLLP find_dnsbll(CONFIG &dc, char *name);
+static DNSBLLP find_dnsbll(CONFIG &dc, char *name) {
+    dnsbllp_map::iterator i = dc.dnsblls.find(name);
+    if (i == dc.dnsblls.end()) return NULL;
+    return (*i).second;
+}
+
+
+////////////////////////////////////////////////
+//  find the envfrom map with a specific name
+//
+static string_map* find_from_map(CONFIG &dc, char *name);
+static string_map* find_from_map(CONFIG &dc, char *name) {
+    from_map::iterator i = dc.env_from.find(name);
+    if (i == dc.env_from.end()) return NULL;
+    return (*i).second;
+}
+
+
+static string_map& really_find_from_map(CONFIG &dc, char *name);
+static string_map& really_find_from_map(CONFIG &dc, char *name) {
+    string_map *sm = find_from_map(dc, name);
+    if (!sm) {
+        sm = new string_map;
+        dc.env_from[name] = sm;
+    }
+    return *sm;
+}
+
+
+
+////////////////////////////////////////////////
+//  read a resolver request from the socket, process it, and
+//  write the result back to the socket.
+
+static void process_resolver_requests(int socket);
+static void process_resolver_requests(int socket) {
+#ifdef NS_MAXDNAME
+    char question[NS_MAXDNAME];
+#else
+    char question[1000];
+#endif
+    glommer glom;
+
+    int maxq = sizeof(question);
+    while (true) {
+        // read a question
+        int rs = 0;
+        while (true) {
+            int ns = read(socket, question+rs, maxq-rs);
+            if (ns > 0) {
+                rs += ns;
+                if (question[rs-1] == '\0') {
+                    // last byte read was the null terminator, we are done
+                    break;
+                }
+            }
+            else {
+                // peer closed the socket
+              //my_syslog("!!child worker process, peer closed socket while reading question");
+                shutdown(socket, SHUT_RDWR);
+                close(socket);
+                return;
+            }
+        }
+
+        // find the answer
+#ifdef NS_PACKETSZ
+      //char text[1000];
+      //snprintf(text, sizeof(text), "!!child worker process has a question %s", question);
+      //my_syslog(text);
+        glom.length = res_search(question, ns_c_in, ns_t_a, glom.answer, sizeof(glom.answer));
+        if (glom.length < 0) glom.length = 0;   // represent all errors as zero length answers
+#else
+        glom.length = sizeof(glom.answer);
+        glom.answer = 0;
+        struct hostent *host = gethostbyname(question);
+        if (host && (host->h_addrtype == AF_INET)) {
+            memcpy(&glom.answer, host->h_addr, sizeof(glom.answer));
+        }
+#endif
+
+        // write the answer
+        char *buf = (char *)&glom;
+        int   len = glom.length + sizeof(glom.length);
+      //snprintf(text, sizeof(text), "!!child worker process writing answer length %d for total %d", glom.length, len);
+      //my_syslog(text);
+        int    ws = 0;
+        while (len > ws) {
+            int ns = write(socket, buf+ws, len-ws);
+            if (ns > 0) {
+                ws += ns;
+            }
+            else {
+                // peer closed the socket!
+              //my_syslog("!!child worker process, peer closed socket while writing answer");
+                shutdown(socket, SHUT_RDWR);
+                close(socket);
+                return;
+            }
+        }
+    }
+}
+
+
+////////////////////////////////////////////////
+//  ask a dns question and get an A record answer - we don't try
+//  very hard, just using the default resolver retry settings.
+//  If we cannot get an answer, we just accept the mail.
+//
+//
+static int dns_interface(mlfiPriv &priv, char *question, bool maybe_ip, ns_map *nameservers);
+static int dns_interface(mlfiPriv &priv, char *question, bool maybe_ip, ns_map *nameservers) {
+    // this part can be done without locking the resolver mutex. Each
+    // milter thread is talking over its own socket to a separate resolver
+    // process, which does the actual dns resolution.
+    if (priv.err) return 0; // cannot ask more questions on this socket.
+    priv.my_write(question, strlen(question)+1);   // write the question including the null terminator
+    glommer glom;
+    char *buf = (char *)&glom;
+    priv.my_read(buf, sizeof(glom.length));
+    buf += sizeof(glom.length);
+ ///char text[1000];
+ ///snprintf(text, sizeof(text), "!!milter thread wrote question %s and has answer length %d", question, glom.length);
+ ///my_syslog(text);
+    if ((glom.length < 0) || (glom.length > sizeof(glom.answer))) {
+        priv.err = true;
+        return 0;  // cannot process overlarge answers
+    }
+    priv.my_read(buf, glom.length);
+
+#ifdef NS_PACKETSZ
+    // now we need to lock the resolver mutex to keep the milter threads from
+    // stepping on each other while parsing the dns answer.
+    int ret_address = 0;
+    pthread_mutex_lock(&resolve_mutex);
+        if (glom.length > 0) {
+            // parse the answer
+            ns_msg handle;
+            ns_rr  rr;
+            if (ns_initparse(glom.answer, glom.length, &handle) == 0) {
+                // look for ns names
+                if (nameservers) {
+                    ns_map &ns = *nameservers;
+                    int rrnum = 0;
+                    while (ns_parserr(&handle, ns_s_ns, rrnum++, &rr) == 0) {
+                        if (ns_rr_type(rr) == ns_t_ns) {
+                            char nam[NS_MAXDNAME+1];
+                            char         *n = nam;
+                            const u_char *p = ns_rr_rdata(rr);
+                            while (((n-nam) < NS_MAXDNAME) && ((p-glom.answer) < glom.length) && *p) {
+                                size_t s = *(p++);
+                                if (s > 191) {
+                                    // compression pointer
+                                    s = (s-192)*256 + *(p++);
+                                    if (s >= glom.length) break; // pointer outside bounds of answer
+                                    p = glom.answer + s;
+                                    s = *(p++);
+                                }
+                                if (s > 0) {
+                                    if ((n-nam)         >= (NS_MAXDNAME-s)) break;  // destination would overflow name buffer
+                                    if ((p-glom.answer) >= (glom.length-s)) break;  // source outside bounds of answer
+                                    memcpy(n, p, s);
+                                    n += s;
+                                    p += s;
+                                    *(n++) = '.';
+                                }
+                            }
+                            if (n-nam) n--;             // remove trailing .
+                            *n = '\0';                  // null terminate it
+                            register_string(ns, nam, question);     // ns host to lookup later
+                        }
+                    }
+                    rrnum = 0;
+                    while (ns_parserr(&handle, ns_s_ar, rrnum++, &rr) == 0) {
+                        if (ns_rr_type(rr) == ns_t_a) {
+                            char* nam = (char*)ns_rr_name(rr);
+                            ns_mapper::iterator i = ns.ns_ip.find(nam);
+                            if (i != ns.ns_ip.end()) {
+                                // we want this ip address
+                                int address;
+                                memcpy(&address, ns_rr_rdata(rr), sizeof(address));
+                                ns.ns_ip[nam] = address;
+                            }
+                        }
+                    }
+                }
+                int rrnum = 0;
+                while (ns_parserr(&handle, ns_s_an, rrnum++, &rr) == 0) {
+                    if (ns_rr_type(rr) == ns_t_a) {
+                        int address;
+                        memcpy(&address, ns_rr_rdata(rr), sizeof(address));
+                        ret_address = address;
+                    }
+                }
+            }
+        }
+        if (maybe_ip && !ret_address) {
+            // might be a bare ip address
+            in_addr ip;
+            if (inet_aton(question, &ip)) {
+                ret_address = ip.s_addr;
+            }
+        }
+    pthread_mutex_unlock(&resolve_mutex);
+    return ret_address;
+#else
+    return glom.answer;
+#endif
+}
+
+
+////////////////////////////////////////////////
+//  check a single dnsbl
+//
+static status check_single(mlfiPriv &priv, int ip, char *suffix);
+static status check_single(mlfiPriv &priv, int ip, char *suffix) {
+    // make a dns question
+    const u_char *src = (const u_char *)&ip;
+    if (src[0] == 127) return oksofar;  // don't do dns lookups on localhost
+#ifdef NS_MAXDNAME
+    char question[NS_MAXDNAME];
+#else
+    char question[1000];
+#endif
+    snprintf(question, sizeof(question), "%u.%u.%u.%u.%s.", src[3], src[2], src[1], src[0], suffix);
+    // ask the question, if we get an A record it implies a blacklisted ip address
+    return (dns_interface(priv, question, false, NULL)) ? reject : oksofar;
+}
+
+
+////////////////////////////////////////////////
+//  check a single dnsbl
+//
+static status check_single(mlfiPriv &priv, int ip, DNSBL &bl);
+static status check_single(mlfiPriv &priv, int ip, DNSBL &bl) {
+    return check_single(priv, ip, bl.suffix);
+}
+
+
+////////////////////////////////////////////////
+//  check the dnsbls specified for this recipient
+//
+static status check_dnsbl(mlfiPriv &priv, DNSBLLP dnsbllp, DNSBLP &rejectlist);
+static status check_dnsbl(mlfiPriv &priv, DNSBLLP dnsbllp, DNSBLP &rejectlist) {
+    if (priv.authenticated) return oksofar;
+    if (!dnsbllp)           return oksofar;
+    DNSBLL &dnsbll = *dnsbllp;
+    for (DNSBLL::iterator i=dnsbll.begin(); i!=dnsbll.end(); i++) {
+        DNSBLP dp = *i;     // non null by construction
+        status st;
+        map<DNSBLP, status>::iterator f = priv.checked.find(dp);
+        if (f == priv.checked.end()) {
+            // have not checked this list yet
+            st = check_single(priv, priv.ip, *dp);
+            rejectlist = dp;
+            priv.checked[dp] = st;
+        }
+        else {
+            st = (*f).second;
+            rejectlist = (*f).first;
+        }
+        if (st == reject) return st;
+    }
+    return oksofar;
+}
+
+
+////////////////////////////////////////////////
+//  check the hosts from the body against the content dnsbl
+//
+static status check_hosts(mlfiPriv &priv, char *&host, int &ip);
+static status check_hosts(mlfiPriv &priv, char *&host, int &ip) {
+    CONFIG &dc = *priv.pc;
+    int count = 0;
+    ns_map nameservers;
+    bool ran = priv.pc->host_random;
+    int  lim = priv.pc->host_limit;         // we should not look at more than this many hosts
+    int  cnt = priv.memory->hosts.size();   // number of hosts we could look at
+    int_set ips;    // remove duplicate ip addresses
+    for (string_set::iterator i=priv.memory->hosts.begin(); i!=priv.memory->hosts.end(); i++) {
+        host = *i;  // a reference into priv.memory->hosts, which will live until this smtp transaction is closed
+
+        // don't bother looking up hosts on the ignore list
+        string_set::iterator j = priv.pc->content_host_ignore.find(host);
+        if (j != priv.pc->content_host_ignore.end()) continue;
+
+        // try to only look at lim/cnt fraction of the available cnt host names in random mode
+        if ((cnt > lim) && (lim > 0) && ran) {
+            int r = rand() % cnt;
+            if (r >= lim) {
+                char buf[1000];
+                snprintf(buf, sizeof(buf), "host %s skipped", host);
+                my_syslog(&priv, buf);
+                continue;
+            }
+        }
+        count++;
+        if ((count > lim) && (lim > 0) && (!ran)) {
+            discard(nameservers);
+            return reject_host;
+        }
+        ip = dns_interface(priv, host, true, &nameservers);
+        if (debug_syslog) {
+            char buf[1000];
+            if (ip) {
+                char adr[sizeof "255.255.255.255"];
+                adr[0] = '\0';
+                inet_ntop(AF_INET, (const u_char *)&ip, adr, sizeof(adr));
+                snprintf(buf, sizeof(buf), "host %s found at %s", host, adr);
+            }
+            else {
+                snprintf(buf, sizeof(buf), "host %s not found", host);
+            }
+            my_syslog(&priv, buf);
+        }
+        if (ip) {
+            int_set::iterator i = ips.find(ip);
+            if (i == ips.end()) {
+                ips.insert(ip);
+                status st = check_single(priv, ip, dc.content_suffix);
+                if (st == reject) {
+                    discard(nameservers);
+                    return st;
+                }
+            }
+        }
+    }
+    lim *= 4;   // allow average of 3 ns per host name
+    for (ns_mapper::iterator i=nameservers.ns_ip.begin(); i!=nameservers.ns_ip.end(); i++) {
+        count++;
+        if ((count > lim) && (lim > 0)) {
+            if (ran) continue; // don't complain
+            discard(nameservers);
+            return reject_host;
+        }
+        host = (*i).first;  // a transient reference that needs to be replaced before we return it
+        ip   = (*i).second;
+        if (!ip) ip = dns_interface(priv, host, false, NULL);
+        if (debug_syslog) {
+            char buf[200];
+            if (ip) {
+                char adr[sizeof "255.255.255.255"];
+                adr[0] = '\0';
+                inet_ntop(AF_INET, (const u_char *)&ip, adr, sizeof(adr));
+                snprintf(buf, sizeof(buf), "ns %s found at %s", host, adr);
+            }
+            else {
+                snprintf(buf, sizeof(buf), "ns %s not found", host);
+            }
+            my_syslog(&priv, buf);
+        }
+        if (ip) {
+            int_set::iterator i = ips.find(ip);
+            if (i == ips.end()) {
+                ips.insert(ip);
+                status st = check_single(priv, ip, dc.content_suffix);
+                if (st == reject) {
+                    string_map::iterator j = nameservers.ns_host.find(host);
+                    if (j != nameservers.ns_host.end()) {
+                        char *refer = (*j).second;
+                        char buf[1000];
+                        snprintf(buf, sizeof(buf), "%s with nameserver %s", refer, host);
+                        host = register_string(priv.memory->hosts, buf);    // put a copy into priv.memory->hosts, and return that reference
+                    }
+                    else {
+                        host = register_string(priv.memory->hosts, host);   // put a copy into priv.memory->hosts, and return that reference
+                    }
+                    discard(nameservers);
+                    return st;
+                }
+            }
+        }
+    }
+    discard(nameservers);
+    host = NULL;
+    int bin = priv.memory->binary_tags;
+    int bad = priv.memory->bad_html_tags;
+    lim = priv.pc->tag_limit;
+    if (3*bin > bad) return oksofar;    // probably .zip or .tar.gz with random content
+    if ((bad > lim) && (lim > 0)) return reject_tag;
+    return oksofar;
+}
+
+
+////////////////////////////////////////////////
+// start of sendmail milter interfaces
+//
+sfsistat mlfi_connect(SMFICTX *ctx, char *hostname, _SOCK_ADDR *hostaddr)
+{
+    // allocate some private memory
+    mlfiPriv *priv = new mlfiPriv;
+    if (hostaddr->sa_family == AF_INET) {
+        priv->ip = ((struct sockaddr_in *)hostaddr)->sin_addr.s_addr;
+    }
+
+    // save the private data
+    smfi_setpriv(ctx, (void*)priv);
+
+    // continue processing
+    return SMFIS_CONTINUE;
+}
+
+sfsistat mlfi_envfrom(SMFICTX *ctx, char **from)
+{
+    mlfiPriv &priv = *MLFIPRIV;
+    priv.mailaddr      = strdup(from[0]);
+    priv.authenticated = (smfi_getsymval(ctx, "{auth_authen}") != NULL);
+    return SMFIS_CONTINUE;
+}
+
+sfsistat mlfi_envrcpt(SMFICTX *ctx, char **rcpt)
+{
+    DNSBLP rejectlist = NULL;   // list that caused the reject
+    status st = oksofar;
+    mlfiPriv &priv = *MLFIPRIV;
+    CONFIG &dc = *priv.pc;
+    if (!priv.queueid) priv.queueid = strdup(smfi_getsymval(ctx, "i"));
+    char *rcptaddr = rcpt[0];
+    char *dnsname  = lookup(rcptaddr, dc.env_to_dnsbll);
+    char *fromname = lookup(rcptaddr, dc.env_to_chkfrom);
+    if ((strcmp(dnsname,  BLACK) == 0) ||
+        (strcmp(fromname, BLACK) == 0)) {
+        st = black; // two options to blacklist this recipient
+    }
+    else if (strcmp(fromname, WHITE) == 0) {
+        st = white;
+    }
+    else {
+        // check an env_from map
+        string_map *sm = find_from_map(dc, fromname);
+        if (sm != NULL) {
+            fromname = lookup(priv.mailaddr, *sm);  // returns default if name not in map
+            if (strcmp(fromname, BLACK) == 0) {
+                st = black; // blacklist this envelope from value
+            }
+            if (strcmp(fromname, WHITE) == 0) {
+                st = white; // blacklist this envelope from value
+            }
+        }
+    }
+    if ((st == oksofar) && (strcmp(dnsname, WHITE) != 0)) {
+        // check dns lists
+        st = check_dnsbl(priv, find_dnsbll(dc, dnsname), rejectlist);
+    }
+
+    if (st == reject) {
+        // reject the recipient based on some dnsbl
+        char adr[sizeof "255.255.255.255"];
+        adr[0] = '\0';
+        inet_ntop(AF_INET, (const u_char *)&priv.ip, adr, sizeof(adr));
+        char buf[2000];
+        snprintf(buf, sizeof(buf), rejectlist->message, adr, adr);
+        smfi_setreply(ctx, "550", "5.7.1", buf);
+        return SMFIS_REJECT;
+    }
+    else if (st == black) {
+        // reject the recipient based on blacklisting either from or to
+        smfi_setreply(ctx, "550", "5.7.1", "no such user");
+        return SMFIS_REJECT;
+    }
+    else {
+        // accept the recipient
+        if (st == oksofar) {
+            // but remember the non-whites
+            register_string(priv.non_whites, rcptaddr);
+            priv.only_whites = false;
+        }
+        if (st == white) {
+            priv.have_whites = true;
+        }
+        return SMFIS_CONTINUE;
+    }
+}
+
+sfsistat mlfi_body(SMFICTX *ctx, u_char *data, size_t len)
+{
+    mlfiPriv &priv = *MLFIPRIV;
+    if (priv.authenticated)       return SMFIS_CONTINUE;
+    if (priv.only_whites)         return SMFIS_CONTINUE;
+    if (!priv.pc->content_suffix) return SMFIS_CONTINUE;
+    priv.scanner->scan(data, len);
+    return SMFIS_CONTINUE;
+}
+
+sfsistat mlfi_eom(SMFICTX *ctx)
+{
+    sfsistat  rc;
+    mlfiPriv &priv = *MLFIPRIV;
+    char     *host = NULL;
+    int       ip;
+    status    st;
+    // process end of message
+    if (priv.authenticated         ||
+        priv.only_whites           ||
+        (!priv.pc->content_suffix) ||
+        ((st=check_hosts(priv, host, ip)) == oksofar)) rc = SMFIS_CONTINUE;
+    else {
+        if (!priv.have_whites) {
+            // can reject the entire message
+            char buf[2000];
+            if (st == reject_tag) {
+                // rejected due to excessive bad html tags
+                snprintf(buf, sizeof(buf), priv.pc->tag_limit_message);
+            }
+            else if (st == reject_host) {
+                // rejected due to excessive unique host/urls
+                snprintf(buf, sizeof(buf), priv.pc->host_limit_message);
+            }
+            else {
+                char adr[sizeof "255.255.255.255"];
+                adr[0] = '\0';
+                inet_ntop(AF_INET, (const u_char *)&ip, adr, sizeof(adr));
+                snprintf(buf, sizeof(buf), priv.pc->content_message, host, adr);
+            }
+            smfi_setreply(ctx, "550", "5.7.1", buf);
+            rc = SMFIS_REJECT;
+        }
+        else {
+            // need to accept it but remove the recipients that don't want it
+            for (string_set::iterator i=priv.non_whites.begin(); i!=priv.non_whites.end(); i++) {
+                char *rcpt = *i;
+                smfi_delrcpt(ctx, rcpt);
+            }
+            rc = SMFIS_CONTINUE;
+        }
+    }
+    // reset for a new message on the same connection
+    mlfi_abort(ctx);
+    return rc;
+}
+
+sfsistat mlfi_abort(SMFICTX *ctx)
+{
+    mlfiPriv &priv = *MLFIPRIV;
+    priv.reset();
+    return SMFIS_CONTINUE;
+}
+
+sfsistat mlfi_close(SMFICTX *ctx)
+{
+    mlfiPriv *priv = MLFIPRIV;
+    if (!priv) return SMFIS_CONTINUE;
+    delete priv;
+    smfi_setpriv(ctx, NULL);
+    return SMFIS_CONTINUE;
+}
+
+struct smfiDesc smfilter =
+{
+    "DNSBL",            // filter name
+    SMFI_VERSION,       // version code -- do not change
+    SMFIF_DELRCPT,      // flags
+    mlfi_connect,       // connection info filter
+    NULL,               // SMTP HELO command filter
+    mlfi_envfrom,       // envelope sender filter
+    mlfi_envrcpt,       // envelope recipient filter
+    NULL,               // header filter
+    NULL,               // end of header
+    mlfi_body,          // body block filter
+    mlfi_eom,           // end of message
+    mlfi_abort,         // message aborted
+    mlfi_close,         // connection cleanup
+};
+
+
+////////////////////////////////////////////////
+//  reload the config
+//
+static CONFIG* new_conf();
+static CONFIG* new_conf() {
+    CONFIG *newc = new CONFIG;
+    pthread_mutex_lock(&config_mutex);
+        newc->generation = generation++;
+    pthread_mutex_unlock(&config_mutex);
+    char buf[200];
+    snprintf(buf, sizeof(buf), "loading configuration generation %d", newc->generation);
+    my_syslog(buf);
+    if (load_conf(*newc, "dnsbl.conf") {
+        newc->load_time = time(NULL);
+        return newc;
+    }
+    delete newc;
+    return NULL;
+}
+
+
+////////////////////////////////////////////////
+//  thread to watch the old config files for changes
+//  and reload when needed. we also cleanup old
+//  configs whose reference count has gone to zero.
+//
+static void* config_loader(void *arg);
+static void* config_loader(void *arg) {
+    typedef set<CONFIG *> configp_set;
+    configp_set old_configs;
+    while (loader_run) {
+        sleep(180);  // look for modifications every 3 minutes
+        if (!loader_run) break;
+        CONFIG &dc = *config;
+        time_t then = dc.load_time;
+        struct stat st;
+        bool reload = false;
+        for (string_list::iterator i=dc.config_files.begin(); i!=dc.config_files.end(); i++) {
+            char *fn = *i;
+            if (stat(fn, &st))           reload = true; // file disappeared
+            else if (st.st_mtime > then) reload = true; // file modified
+            if (reload) break;
+        }
+        if (reload) {
+            CONFIG *newc = new_conf();
+            // replace the global config pointer
+            pthread_mutex_lock(&config_mutex);
+                CONFIG *old = config;
+                config = newc;
+            pthread_mutex_unlock(&config_mutex);
+            if (old) old_configs.insert(old);
+        }
+        // now look for old configs with zero ref counts
+        for (configp_set::iterator i=old_configs.begin(); i!=old_configs.end(); ) {
+            CONFIG *old = *i;
+            if (!old->reference_count) {
+                char buf[200];
+                snprintf(buf, sizeof(buf), "freeing memory for old configuration generation %d", old->generation);
+                my_syslog(buf);
+                delete old; // destructor does all the work
+                old_configs.erase(i++);
+            }
+            else i++;
+        }
+    }
+    return NULL;
+}
+
+
+static void usage(char *prog);
+static void usage(char *prog)
+{
+    fprintf(stderr, "Usage: %s  [-d] [-c] -r port -p sm-sock-addr [-t timeout]\n", prog);
+    fprintf(stderr, "where port is for the connection to our own dns resolver processes\n");
+    fprintf(stderr, "    and should be local-domain-socket-file-name\n");
+    fprintf(stderr, "where sm-sock-addr is for the connection to sendmail\n");
+    fprintf(stderr, "    and should be one of\n");
+    fprintf(stderr, "        inet:port@ip-address\n");
+    fprintf(stderr, "        local:local-domain-socket-file-name\n");
+    fprintf(stderr, "-c will load and dump the config to stdout\n");
+    fprintf(stderr, "-d will add some syslog debug messages\n");
+}
+
+
+
+static void setup_socket(char *sock);
+static void setup_socket(char *sock) {
+    unlink(sock);
+    //  sockaddr_un addr;
+    //  memset(&addr, '\0', sizeof addr);
+    //  addr.sun_family = AF_UNIX;
+    //  strncpy(addr.sun_path, sock, sizeof(addr.sun_path)-1);
+    //  int s = socket(AF_UNIX, SOCK_STREAM, 0);
+    //  bind(s, (sockaddr*)&addr, sizeof(addr));
+    //  close(s);
+}
+
+
+/*
+ * The signal handler function -- only gets called when a SIGCHLD
+ * is received, ie when a child terminates
+ */
+void sig_chld(int signo)
+{
+    int status;
+    /* Wait for any child without blocking */
+    while (waitpid(-1, &status, WNOHANG) > 0) {
+        // ignore child exit status, we only do this to cleanup zombies
+    }
+}
+
+
+int main(int argc, char**argv)
+{
+    token_init();
+    bool check   = false;
+    bool setconn = false;
+    bool setreso = false;
+    int c;
+    const char *args = "r:p:t:hcd";
+    extern char *optarg;
+
+    // Process command line options
+    while ((c = getopt(argc, argv, args)) != -1) {
+        switch (c) {
+            case 'r':
+                if (optarg == NULL || *optarg == '\0') {
+                    fprintf(stderr, "Illegal resolver socket: %s\n", optarg);
+                    exit(EX_USAGE);
+                }
+                resolver_port = strdup(optarg);
+                setup_socket(resolver_port);
+                setreso = true;
+                break;
+
+            case 'p':
+                if (optarg == NULL || *optarg == '\0') {
+                    fprintf(stderr, "Illegal sendmail socket: %s\n", optarg);
+                    exit(EX_USAGE);
+                }
+                if (smfi_setconn(optarg) == MI_FAILURE) {
+                    fprintf(stderr, "smfi_setconn failed\n");
+                    exit(EX_SOFTWARE);
+                }
+                     if (strncasecmp(optarg, "unix:", 5) == 0)  setup_socket(optarg + 5);
+                else if (strncasecmp(optarg, "local:", 6) == 0) setup_socket(optarg + 6);
+                setconn = true;
+                break;
+
+            case 't':
+                if (optarg == NULL || *optarg == '\0') {
+                    fprintf(stderr, "Illegal timeout: %s\n", optarg);
+                    exit(EX_USAGE);
+                }
+                if (smfi_settimeout(atoi(optarg)) == MI_FAILURE) {
+                    fprintf(stderr, "smfi_settimeout failed\n");
+                    exit(EX_SOFTWARE);
+                }
+                break;
+
+            case 'c':
+                check = true;
+                break;
+
+            case 'd':
+                debug_syslog = true;
+                break;
+
+            case 'h':
+            default:
+                usage(argv[0]);
+                exit(EX_USAGE);
+        }
+    }
+
+    if (check) {
+        CONFIG *conf = new_conf();
+        if (conf) {
+            conf->dump();
+            delete conf;
+            return 0;
+        }
+        else {
+            return 1;   // config failed to load
+        }
+    }
+
+    if (!setconn) {
+        fprintf(stderr, "%s: Missing required -p argument\n", argv[0]);
+        usage(argv[0]);
+        exit(EX_USAGE);
+    }
+
+    if (!setreso) {
+        fprintf(stderr, "%s: Missing required -r argument\n", argv[0]);
+        usage(argv[0]);
+        exit(EX_USAGE);
+    }
+
+    if (smfi_register(smfilter) == MI_FAILURE) {
+        fprintf(stderr, "smfi_register failed\n");
+        exit(EX_UNAVAILABLE);
+    }
+
+    // switch to background mode
+    if (daemon(1,0) < 0) {
+        fprintf(stderr, "daemon() call failed\n");
+        exit(EX_UNAVAILABLE);
+    }
+
+    // write the pid
+    const char *pidpath = "/var/run/dnsbl.pid";
+    unlink(pidpath);
+    FILE *f = fopen(pidpath, "w");
+    if (f) {
+#ifdef linux
+        // from a comment in the DCC source code:
+        // Linux threads are broken.  Signals given the
+        // original process are delivered to only the
+        // thread that happens to have that PID.  The
+        // sendmail libmilter thread that needs to hear
+        // SIGINT and other signals does not, and that breaks
+        // scripts that need to stop milters.
+        // However, signaling the process group works.
+        fprintf(f, "-%d\n", (u_int)getpgrp());
+#else
+        fprintf(f, "%d\n", (u_int)getpid());
+#endif
+        fclose(f);
+    }
+
+    // initialize the thread sync objects
+    pthread_mutex_init(&config_mutex, 0);
+    pthread_mutex_init(&syslog_mutex, 0);
+    pthread_mutex_init(&resolve_mutex, 0);
+    pthread_mutex_init(&fd_pool_mutex, 0);
+
+    // drop root privs
+    struct passwd *pw = getpwnam("dnsbl");
+    if (pw) {
+        if (setgid(pw->pw_gid) == -1) {
+            my_syslog("failed to switch to group dnsbl");
+        }
+        if (setuid(pw->pw_uid) == -1) {
+            my_syslog("failed to switch to user dnsbl");
+        }
+    }
+
+    // fork off the resolver listener process
+    pid_t child = fork();
+    if (child < 0) {
+        my_syslog("failed to create resolver listener process");
+        exit(0);
+    }
+    if (child == 0) {
+        // we are the child - dns resolver listener process
+        resolver_socket = socket(AF_UNIX, SOCK_STREAM, 0);
+        if (resolver_socket < 0) {
+            my_syslog("child failed to create resolver socket");
+            exit(0);   // failed
+        }
+        sockaddr_un server;
+        memset(&server, '\0', sizeof(server));
+        server.sun_family = AF_UNIX;
+        strncpy(server.sun_path, resolver_port, sizeof(server.sun_path)-1);
+        //try to bind the address to the socket.
+        if (bind(resolver_socket, (sockaddr *)&server, sizeof(server)) < 0) {
+            // bind failed
+            shutdown(resolver_socket, SHUT_RDWR);
+            close(resolver_socket);
+            my_syslog("child failed to bind resolver socket");
+            exit(0);   // failed
+        }
+        //listen on the socket.
+        if (listen(resolver_socket, 10) < 0) {
+            // listen failed
+            shutdown(resolver_socket, SHUT_RDWR);
+            close(resolver_socket);
+            my_syslog("child failed to listen to resolver socket");
+            exit(0);   // failed
+        }
+        // setup sigchld handler to prevent zombies
+        struct sigaction act;
+        act.sa_handler = sig_chld;      // Assign sig_chld as our SIGCHLD handler
+        sigemptyset(&act.sa_mask);      // We don't want to block any other signals in this example
+        act.sa_flags = SA_NOCLDSTOP;    // only want children that have terminated
+        if (sigaction(SIGCHLD, &act, NULL) < 0) {
+            my_syslog("child failed to setup SIGCHLD handler");
+            exit(0);   // failed
+        }
+        while (true) {
+            sockaddr_un client;
+            socklen_t   clientlen = sizeof(client);
+            int s = accept(resolver_socket, (sockaddr *)&client, &clientlen);
+            if (s > 0) {
+                // accept worked, it did not get cancelled before we could accept it
+                // fork off a process to handle this connection
+                int newchild = fork();
+                if (newchild == 0) {
+                    // this is the worker process
+                    // child does not need the listening socket
+                    close(resolver_socket);
+                  //my_syslog("child forked a worker process");
+                    process_resolver_requests(s);
+                  //my_syslog("child terminated a worker process");
+                    exit(0);
+                }
+                else {
+                    // this is the parent
+                    // parent does not need the accepted socket
+                    close(s);
+                }
+            }
+        }
+        exit(0);    // make sure we don't fall thru.
+    }
+    else {
+        sleep(2);   // allow child to get started
+    }
+
+    // load the initial config
+    config = new_conf();
+
+    // only create threads after the fork() in daemon
+    pthread_t tid;
+    if (pthread_create(&tid, 0, config_loader, 0))
+        my_syslog("failed to create config loader thread");
+    if (pthread_detach(tid))
+        my_syslog("failed to detach config loader thread");
+
+    time_t starting = time(NULL);
+    int rc = smfi_main();
+    if ((rc != MI_SUCCESS) && (time(NULL) > starting+5*60)) {
+        my_syslog("trying to restart after smfi_main()");
+        loader_run = false;     // eventually the config loader thread will terminate
+        execvp(argv[0], argv);
+    }
+    exit((rc == MI_SUCCESS) ? 0 : EX_UNAVAILABLE);
+}
+