# HG changeset patch # User carl # Date 1104806150 28800 # Node ID 510a511ad554915116f3888661a32070d66517c2 # Parent 7bb8bbf792854f075d58207afc5ad541a9c7f620 Add resolver processes to allow better performance on busy machines diff -r 7bb8bbf79285 -r 510a511ad554 ChangeLog --- a/ChangeLog Thu Oct 28 22:54:34 2004 -0700 +++ b/ChangeLog Mon Jan 03 18:35:50 2005 -0800 @@ -1,5 +1,14 @@ $Id$ +4.0 2005-01-03 + Initialize the thread mutex objects early, before they are needed + by possible calls to my_syslog. + + Fork off a separate resolver listener process, so we can do multiple + dns operations in parallel. For each simultaneous inbound email, + we have a separate sendmail process, a milter thread, and a dns + resolver process. + 3.7 2004-10-28 Added an 'ignore' command to the conf file, used to ignore some hosts that might end up on the SBL and otherwise trip the content diff -r 7bb8bbf79285 -r 510a511ad554 dnsbl.rc --- a/dnsbl.rc Thu Oct 28 22:54:34 2004 -0700 +++ b/dnsbl.rc Mon Jan 03 18:35:50 2005 -0800 @@ -22,7 +22,7 @@ echo -n "Starting dnsbl-milter: " if [ ! -f /var/lock/subsys/dnsbl ]; then cd /etc/dnsbl # conf file is here - /usr/sbin/dnsbl -d -p local:/var/run/dnsbl/dnsbl.sock + /usr/sbin/dnsbl -d -r 54 -p local:/var/run/dnsbl/dnsbl.sock RETVAL=$? pid=`pidof -s /usr/sbin/dnsbl` if [ $pid ] diff -r 7bb8bbf79285 -r 510a511ad554 dnsbl.spec.in --- a/dnsbl.spec.in Thu Oct 28 22:54:34 2004 -0700 +++ b/dnsbl.spec.in Mon Jan 03 18:35:50 2005 -0800 @@ -1,6 +1,6 @@ Summary: DNSBL Sendmail Milter Name: dnsbl -Version: 3.7 +Version: 4.0 Release: 2 Copyright: GPL Group: System Environment/Daemons @@ -45,6 +45,7 @@ mkdir -p %{buildroot}/etc/dnsbl install -m 644 dnsbl.conf %{buildroot}/etc/dnsbl/dnsbl.conf +install -m 644 hosts-ignore.conf %{buildroot}/etc/dnsbl/hosts-ignore.conf install -m 644 html-tags.conf %{buildroot}/etc/dnsbl/html-tags.conf install -m 644 tld.conf %{buildroot}/etc/dnsbl/tld.conf install -m 644 sample.conf %{buildroot}/etc/dnsbl/sample.conf @@ -105,6 +106,10 @@ %dir %attr(0750,dnsbl,root) /var/run/dnsbl %changelog +* Tue Jan 03 2005 Carl Byington 1.4 +- added hosts-ignore conf file +- see RELEASE_NOTES + * Thu Jul 15 2004 John Gunkel 1.3 - patch to rc file no longer needed - see RELEASE_NOTES diff -r 7bb8bbf79285 -r 510a511ad554 package.bash --- a/package.bash Thu Oct 28 22:54:34 2004 -0700 +++ b/package.bash Mon Jan 03 18:35:50 2005 -0800 @@ -1,6 +1,6 @@ #!/bin/bash -VER=dnsbl-3.7 +VER=dnsbl-4.0 mkdir $VER target1=/home/httpd/html/510sg/util/dnsbl.tar.gz target2=/home/httpd/html/510sg/dnsbl.conf diff -r 7bb8bbf79285 -r 510a511ad554 sendmail.st Binary file sendmail.st has changed diff -r 7bb8bbf79285 -r 510a511ad554 src/dnsbl.cpp --- a/src/dnsbl.cpp Thu Oct 28 22:54:34 2004 -0700 +++ b/src/dnsbl.cpp Mon Jan 03 18:35:50 2005 -0800 @@ -1,12 +1,13 @@ /* -Copyright (c) 2004 Carl Byington - 510 Software Group, released under -the GPL version 2 or any later version at your choice available at +Copyright (c) 2004, 2005 Carl Byington - 510 Software Group, released +under the GPL version 2 or any later version at your choice available at http://www.fsf.org/licenses/gpl.txt Based on a sample milter Copyright (c) 2000-2003 Sendmail, Inc. and its suppliers. Inspired by the DCC by Rhyolite Software +-r port The port used to talk to our internal dns resolver processes -p port The port through which the MTA will connect to this milter. -t sec The timeout value. -c Check the config, and print a copy to stdout. Don't start the @@ -27,6 +28,10 @@ 4) Check if the envelope from domain name primary MX points 127.0.0.0/8 +5) Add option for using smtp connections to verify addresses from backup +mx machines. This allows the backup mx to learn the valid addresses +on the primary machine. + */ @@ -192,6 +197,23 @@ static pthread_mutex_t config_mutex; static pthread_mutex_t syslog_mutex; static pthread_mutex_t resolve_mutex; +static pthread_mutex_t fd_pool_mutex; +static std::set fd_pool; + +static int NULL_SOCKET = -1; +static int resolver_port = 0; // global port number to talk to the dns resolver process +static int resolver_socket = NULL_SOCKET; // socket used to listen for resolver requests +static time_t ERROR_SOCKET_TIME = 60; // number of seconds between attempts to open the spam filter socket +static time_t last_error_time; + +#ifdef NS_PACKETSZ + // packed structure to allow a single socket write to dump the + // length and the following answer. The packing attribute is gcc specific. + struct glommer { + int length; + u_char answer[NS_PACKETSZ]; + } __attribute__ ((packed)); +#endif struct mlfiPriv; @@ -259,12 +281,61 @@ //////////////////////////////////////////////// +// disconnect the fd from the dns resolver process +// +void my_disconnect(int sock); +void my_disconnect(int sock) +{ + if (sock != NULL_SOCKET) { + shutdown(sock, SHUT_RDWR); + close(sock); + } +} + + +//////////////////////////////////////////////// +// return fd connected to the dns resolver process +// +int my_connect(); +int my_connect() +{ + // if we have had recent errors, don't even try to open the socket + time_t now = time(NULL); + if ((now - last_error_time) < ERROR_SOCKET_TIME) return NULL_SOCKET; + + // nothing recent, maybe this time it will work + int sock = NULL_SOCKET; + hostent *host = gethostbyname("localhost"); + if (host) { + sockaddr_in server; + server.sin_family = host->h_addrtype; + server.sin_port = htons(resolver_port); + memcpy(&server.sin_addr, host->h_addr_list[0], host->h_length); + sock = socket(PF_INET, SOCK_STREAM, 0); + if (sock != NULL_SOCKET) { + bool rc = (connect(sock, (sockaddr *)&server, sizeof(server)) == 0); + if (!rc) { + int er = errno; + my_disconnect(sock); + sock = NULL_SOCKET; + last_error_time = now; + } + } + else last_error_time = now; + } + return sock; +} + + +//////////////////////////////////////////////// // mail filter private data, held for us by sendmail // struct mlfiPriv { // connection specific data CONFIG *pc; // global context with our maps + int fd; // to talk to dns resolvers process + bool err; // did we get any errors on the resolver socket? int ip; // ip4 address of the smtp client map checked; // status from those lists // message specific data @@ -279,12 +350,17 @@ mlfiPriv(); ~mlfiPriv(); void reset(bool final = false); // for a new message + void get_fd(); + void return_fd(); + int my_read(char *buf, int len); + int my_write(char *buf, int len); }; mlfiPriv::mlfiPriv() { pthread_mutex_lock(&config_mutex); pc = config; pc->reference_count++; pthread_mutex_unlock(&config_mutex); + get_fd(); ip = 0; mailaddr = NULL; queueid = NULL; @@ -295,6 +371,7 @@ scanner = new url_scanner(memory); } mlfiPriv::~mlfiPriv() { + return_fd(); pthread_mutex_lock(&config_mutex); pc->reference_count--; pthread_mutex_unlock(&config_mutex); @@ -317,6 +394,96 @@ } } +void mlfiPriv::get_fd() +{ + err = true; + fd = NULL_SOCKET; + int result = pthread_mutex_lock(&fd_pool_mutex); + if (!result) { + std::set::iterator i; + i = fd_pool.begin(); + if (i != fd_pool.end()) { + // have at least one fd in the pool + err = false; + fd = *i; + fd_pool.erase(fd); + } + else { + // pool is empty, get a new fd + fd = my_connect(); + err = (fd == NULL_SOCKET); + } + pthread_mutex_unlock(&fd_pool_mutex); + } + else { + // cannot lock the pool, just get a new fd + fd = my_connect(); + err = (fd == NULL_SOCKET); + } +} + +void mlfiPriv::return_fd() +{ + if (err) { + // this fd got a socket error, so close it, rather than returning it to the pool + my_disconnect(fd); + } + else { + int result = pthread_mutex_lock(&fd_pool_mutex); + if (!result) { + // return the fd to the pool + fd_pool.insert(fd); + pthread_mutex_unlock(&fd_pool_mutex); + } + else { + // could not lock the pool, so just close the fd + my_disconnect(fd); + } + } +} + +int mlfiPriv::my_write(char *buf, int len) +{ + if (err) return 0; + int rs = 0; + while (len) { + int ws = write(fd, buf, len); + if (ws > 0) { + rs += ws; + len -= ws; + buf += ws; + } + else { + // peer closed the socket! + rs = 0; + err = true; + break; + } + } + return rs; +} + +int mlfiPriv::my_read(char *buf, int len) +{ + if (err) return 0; + int rs = 0; + while (len > 1) { + int ws = read(fd, buf, len); + if (ws > 0) { + rs += ws; + len -= ws; + buf += ws; + } + else { + // peer closed the socket! + rs = 0; + err = true; + break; + } + } + return rs; +} + #define MLFIPRIV ((struct mlfiPriv *) smfi_getpriv(ctx)) @@ -438,23 +605,109 @@ //////////////////////////////////////////////// -// +// read a resolver request from the socket, process it, and +// write the result back to the socket. + +#ifdef NS_PACKETSZ +static void process_resolver_requests(int socket); +static void process_resolver_requests(int socket) { +#ifdef NS_MAXDNAME + char question[NS_MAXDNAME]; +#else + char question[1000]; +#endif + glommer glom; + + int maxq = sizeof(question); + while (true) { + // read a question + int rs = 0; + while (true) { + int ns = read(socket, question+rs, maxq-rs); + if (ns > 0) { + rs += ns; + if (question[rs-1] == '\0') { + // last byte read was the null terminator, we are done + break; + } + } + else { + // peer closed the socket + //my_syslog("child worker process, peer closed socket while reading question"); + shutdown(socket, SHUT_RDWR); + close(socket); + return; + } + } + + // find the answer + //char text[1000]; + //snprintf(text, sizeof(text), "child worker process has a question %s", question); + //my_syslog(text); + glom.length = res_search(question, ns_c_in, ns_t_a, glom.answer, sizeof(glom.answer)); + if (glom.length < 0) glom.length = 0; // represent all errors as zero length answers + + // write the answer + char *buf = (char *)&glom; + int len = glom.length + sizeof(glom.length); + //snprintf(text, sizeof(text), "child worker process writing answer length %d for total %d", glom.length, len); + //my_syslog(text); + int ws = 0; + while (len > ws) { + int ns = write(socket, buf+ws, len-ws); + if (ns > 0) { + ws += ns; + } + else { + // peer closed the socket! + //my_syslog("child worker process, peer closed socket while writing answer"); + shutdown(socket, SHUT_RDWR); + close(socket); + return; + } + } + } +} +#endif + + +//////////////////////////////////////////////// // ask a dns question and get an A record answer - we don't try // very hard, just using the default resolver retry settings. -// If we cannot get an answer, we just accept the mail. The -// caller must ensure thread safety. +// If we cannot get an answer, we just accept the mail. // // -static int dns_interface(char *question, bool maybe_ip, ns_map *nameservers); -static int dns_interface(char *question, bool maybe_ip, ns_map *nameservers) { +static int dns_interface(mlfiPriv &priv, char *question, bool maybe_ip, ns_map *nameservers); +static int dns_interface(mlfiPriv &priv, char *question, bool maybe_ip, ns_map *nameservers) { + int ret_address = 0; #ifdef NS_PACKETSZ - u_char answer[NS_PACKETSZ]; - int length = res_search(question, ns_c_in, ns_t_a, answer, sizeof(answer)); - if (length >= 0) { // no error yet + + // this part can be done without locking the resolver mutex. Each + // milter thread is talking over its own socket to a separate resolver + // process, which does the actual dns resolution. + if (priv.err) return 0; // cannot ask more questions on this socket. + priv.my_write(question, strlen(question)+1); // write the question including the null terminator + glommer glom; + char *buf = (char *)&glom; + priv.my_read(buf, sizeof(glom.length)); + buf += sizeof(glom.length); + //char text[1000]; + //snprintf(text, sizeof(text), "milter thread wrote question %s and has answer length %d", question, glom.length); + //my_syslog(text); + if ((glom.length < 0) || (glom.length > sizeof(glom.answer))) { + priv.err = true; + return 0; // cannot process overlarge answers + } + priv.my_read(buf, glom.length); + + // now we need to lock the resolver mutex to keep the milter threads from + // stepping on each other while parsing the dns answer. + pthread_mutex_lock(&resolve_mutex); + if (glom.length > 0) { // parse the answer ns_msg handle; ns_rr rr; - if (ns_initparse(answer, length, &handle) == 0) { + if (ns_initparse(glom.answer, glom.length, &handle) == 0) { // look for ns names if (nameservers) { ns_map &ns = *nameservers; @@ -464,18 +717,18 @@ char nam[NS_MAXDNAME+1]; char *n = nam; const u_char *p = ns_rr_rdata(rr); - while (((n-nam) < NS_MAXDNAME) && ((p-answer) < length) && *p) { + while (((n-nam) < NS_MAXDNAME) && ((p-glom.answer) < glom.length) && *p) { size_t s = *(p++); if (s > 191) { // compression pointer s = (s-192)*256 + *(p++); - if (s >= length) break; // pointer outside bounds of answer - p = answer + s; + if (s >= glom.length) break; // pointer outside bounds of answer + p = glom.answer + s; s = *(p++); } if (s > 0) { if ((n-nam) >= (NS_MAXDNAME-s)) break; // destination would overflow name buffer - if ((p-answer) >= (length-s)) break; // source outside bounds of answer + if ((p-glom.answer) >= (glom.length-s)) break; // source outside bounds of answer memcpy(n, p, s); n += s; p += s; @@ -506,44 +759,39 @@ if (ns_rr_type(rr) == ns_t_a) { int address; memcpy(&address, ns_rr_rdata(rr), sizeof(address)); - return address; + ret_address = address; } } } } - if (maybe_ip) { + if (maybe_ip && !ret_address) { // might be a bare ip address in_addr ip; if (inet_aton(question, &ip)) { - return ip.s_addr; + ret_address = ip.s_addr; } } - return 0; + pthread_mutex_unlock(&resolve_mutex); + return ret_address; + #else + // systems without the resolver interface + pthread_mutex_lock(&resolve_mutex); struct hostent *host = gethostbyname(question); - if (!host) return 0; - if (host->h_addrtype != AF_INET) return 0; - int address; - memcpy(&address, host->h_addr, sizeof(address)); - return address; + if (host && (host->h_addrtype == AF_INET)) { + memcpy(&ret_address, host->h_addr, sizeof(ret_address)); + } + pthread_mutex_unlock(&resolve_mutex); + return ret_address; #endif } -static int protected_dns_interface(char *question, bool maybe_ip, ns_map *nameservers); -static int protected_dns_interface(char *question, bool maybe_ip, ns_map *nameservers) { - int ans; - pthread_mutex_lock(&resolve_mutex); - ans = dns_interface(question, maybe_ip, nameservers); - pthread_mutex_unlock(&resolve_mutex); - return ans; - -} //////////////////////////////////////////////// // check a single dnsbl // -static status check_single(int ip, char *suffix); -static status check_single(int ip, char *suffix) { +static status check_single(mlfiPriv &priv, int ip, char *suffix); +static status check_single(mlfiPriv &priv, int ip, char *suffix) { // make a dns question const u_char *src = (const u_char *)&ip; if (src[0] == 127) return oksofar; // don't do dns lookups on localhost @@ -554,16 +802,16 @@ #endif snprintf(question, sizeof(question), "%u.%u.%u.%u.%s.", src[3], src[2], src[1], src[0], suffix); // ask the question, if we get an A record it implies a blacklisted ip address - return (protected_dns_interface(question, false, NULL)) ? reject : oksofar; + return (dns_interface(priv, question, false, NULL)) ? reject : oksofar; } //////////////////////////////////////////////// // check a single dnsbl // -static status check_single(int ip, DNSBL &bl); -static status check_single(int ip, DNSBL &bl) { - return check_single(ip, bl.suffix); +static status check_single(mlfiPriv &priv, int ip, DNSBL &bl); +static status check_single(mlfiPriv &priv, int ip, DNSBL &bl) { + return check_single(priv, ip, bl.suffix); } @@ -581,7 +829,7 @@ map::iterator f = priv.checked.find(dp); if (f == priv.checked.end()) { // have not checked this list yet - st = check_single(priv.ip, *dp); + st = check_single(priv, priv.ip, *dp); rejectlist = dp; priv.checked[dp] = st; } @@ -629,7 +877,7 @@ discard(nameservers); return reject_host; } - ip = protected_dns_interface(host, true, &nameservers); + ip = dns_interface(priv, host, true, &nameservers); if (debug_syslog) { char buf[1000]; if (ip) { @@ -647,7 +895,7 @@ int_set::iterator i = ips.find(ip); if (i == ips.end()) { ips.insert(ip); - status st = check_single(ip, dc.content_suffix); + status st = check_single(priv, ip, dc.content_suffix); if (st == reject) { discard(nameservers); return st; @@ -665,7 +913,7 @@ } host = (*i).first; // a transient reference that needs to be replaced before we return it ip = (*i).second; - if (!ip) ip = protected_dns_interface(host, false, NULL); + if (!ip) ip = dns_interface(priv, host, false, NULL); if (debug_syslog) { char buf[200]; if (ip) { @@ -683,7 +931,7 @@ int_set::iterator i = ips.find(ip); if (i == ips.end()) { ips.insert(ip); - status st = check_single(ip, dc.content_suffix); + status st = check_single(priv, ip, dc.content_suffix); if (st == reject) { string_map::iterator j = nameservers.ns_host.find(host); if (j != nameservers.ns_host.end()) { @@ -1355,9 +1603,11 @@ static void usage(char *prog); static void usage(char *prog) { - fprintf(stderr, "Usage: %s [-d] [-c] -p socket-addr [-t timeout]\n", prog); - fprintf(stderr, "where socket-addr is for the connection to sendmail and should be one of\n"); - fprintf(stderr, " inet:port@local-ip-address\n"); + fprintf(stderr, "Usage: %s [-d] [-c] -r port -p sm-sock-addr [-t timeout]\n", prog); + fprintf(stderr, "where port is for the connection to our own dns resolver processes\n"); + fprintf(stderr, "where sm-sock-addr is for the connection to sendmail\n"); + fprintf(stderr, " and should be one of\n"); + fprintf(stderr, " inet:port@ip-address\n"); fprintf(stderr, " local:local-domain-socket-file-name\n"); fprintf(stderr, "-c will load and dump the config to stdout\n"); fprintf(stderr, "-d will add some syslog debug messages\n"); @@ -1382,16 +1632,26 @@ { bool check = false; bool setconn = false; + bool setreso = false; int c; - const char *args = "p:t:hcd"; + const char *args = "r:p:t:hcd"; extern char *optarg; // Process command line options while ((c = getopt(argc, argv, args)) != -1) { switch (c) { + case 'r': + if (optarg == NULL || *optarg == '\0') { + fprintf(stderr, "Illegal resolver socket: %s\n", optarg); + exit(EX_USAGE); + } + resolver_port = atoi(optarg); + setreso = true; + break; + case 'p': if (optarg == NULL || *optarg == '\0') { - fprintf(stderr, "Illegal conn: %s\n", optarg); + fprintf(stderr, "Illegal sendmail socket: %s\n", optarg); exit(EX_USAGE); } if (smfi_setconn(optarg) == MI_FAILURE) { @@ -1442,6 +1702,12 @@ exit(EX_USAGE); } + if (!setreso) { + fprintf(stderr, "%s: Missing required -r argument\n", argv[0]); + usage(argv[0]); + exit(EX_USAGE); + } + if (smfi_register(smfilter) == MI_FAILURE) { fprintf(stderr, "smfi_register failed\n"); exit(EX_UNAVAILABLE); @@ -1474,6 +1740,64 @@ fclose(f); } + // initialize the thread sync objects + pthread_mutex_init(&config_mutex, 0); + pthread_mutex_init(&syslog_mutex, 0); + pthread_mutex_init(&resolve_mutex, 0); + pthread_mutex_init(&fd_pool_mutex, 0); + +#ifdef NS_PACKETSZ + // fork off the resolver listener process + pid_t child = fork(); + if (child < 0) { + my_syslog("failed to create resolver listener process"); + exit(0); + } + if (child == 0) { + // we are the child - dns resolver listener process + resolver_socket = socket(PF_INET, SOCK_STREAM, 0); + if (resolver_socket < 0) { + my_syslog("child failed to create resolver socket"); + exit(0); // failed + } + sockaddr_in server; + server.sin_family = AF_INET; + server.sin_port = htons(resolver_port); + server.sin_addr.s_addr = 0; + // set the socket options + int reuse_addr = 1; + linger linger_opt; + linger_opt.l_onoff = 0; // off + linger_opt.l_linger = 0; + setsockopt(resolver_socket, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast(&reuse_addr), sizeof(reuse_addr)); + setsockopt(resolver_socket, SOL_SOCKET, SO_LINGER, reinterpret_cast(&linger_opt), sizeof(linger_opt)); + ///// set nonblocking mode + ///int dummy = 0; + ///int flags = fcntl(resolver_socket, F_GETFL, dummy); + ///if (flags >= 0) fcntl(resolver_socket, F_SETFL, flags | O_NONBLOCK); + //try to bind the address to the socket. + if (bind(resolver_socket, (sockaddr *)&server, sizeof(server)) < 0) { + // bind failed + shutdown(resolver_socket, SHUT_RDWR); + close(resolver_socket); + my_syslog("child failed to bind resolver socket"); + exit(0); // failed + } + + //listen on the socket. + if (listen(resolver_socket, 10) < 0) { + // listen failed + shutdown(resolver_socket, SHUT_RDWR); + close(resolver_socket); + my_syslog("child failed to listen to the resolver socket"); + return -1; + } + } + else { + sleep(2); // allow child to get started + } +#endif + // drop root privs struct passwd *pw = getpwnam("dnsbl"); if (pw) { @@ -1485,10 +1809,28 @@ } } - // initialize the thread sync objects - pthread_mutex_init(&config_mutex, 0); - pthread_mutex_init(&syslog_mutex, 0); - pthread_mutex_init(&resolve_mutex, 0); +#ifdef NS_PACKETSZ + if (child == 0) { + while (true) { + sockaddr_in client; + socklen_t clientlen = sizeof(client); + int s = accept(resolver_socket, (sockaddr *)&client, &clientlen); + if (s > 0) { + // accept worked, it did not get cancelled before we could accept it + // fork off a process to handle this connection + int newchild = fork(); + if (newchild == 0) { + // this is the worker process + my_syslog("child forked a worker process"); + process_resolver_requests(s); + my_syslog("child terminated a worker process"); + exit(0); + } + } + } + exit(0); // make sure we don't fall thru. + } +#endif // load the initial config config = new_conf(); diff -r 7bb8bbf79285 -r 510a511ad554 test.bash --- a/test.bash Thu Oct 28 22:54:34 2004 -0700 +++ b/test.bash Mon Jan 03 18:35:50 2005 -0800 @@ -40,10 +40,10 @@ pid=/var/run/dnsbl.pid echo start the milter mkdir -p /var/run/dnsbl -chmod 750 /var/run/dnsbl +chmod 700 /var/run/dnsbl chown dnsbl:dnsbl /var/run/dnsbl mv -f $pid $pid.save -./dnsbl -d -p local:/var/run/dnsbl/dnsbl.sock +./dnsbl -d -r 54 -p local:/var/run/dnsbl/dnsbl.sock2 sleep 5 P2=`cat $pid` mv -f $pid.save $pid diff -r 7bb8bbf79285 -r 510a511ad554 test.cf --- a/test.cf Thu Oct 28 22:54:34 2004 -0700 +++ b/test.cf Mon Jan 03 18:35:50 2005 -0800 @@ -16,7 +16,7 @@ ##### ##### SENDMAIL CONFIGURATION FILE ##### -##### built by root@ns.five-ten-sg.com on Wed Apr 21 11:01:48 PDT 2004 +##### built by root@ns.five-ten-sg.com on Mon Jan 3 13:23:43 PST 2005 ##### in /usr/src/rh8/gpl/dnsbl ##### using /usr/share/sendmail-cf/ as configuration include directory ##### @@ -1730,7 +1730,7 @@ ###################################################################### ###################################################################### -Xdnsbl, S=local:/var/run/dnsbl/dnsbl.sock, F=T, T=S:30s;R:30s;E:30s +Xdnsbl, S=local:/var/run/dnsbl/dnsbl.sock2, F=T, T=S:30s;R:30s;E:30s # ###################################################################### ###################################################################### diff -r 7bb8bbf79285 -r 510a511ad554 test.mc --- a/test.mc Thu Oct 28 22:54:34 2004 -0700 +++ b/test.mc Mon Jan 03 18:35:50 2005 -0800 @@ -50,6 +50,6 @@ FEATURE(`no_default_msa',`dnl') VIRTUSER_DOMAIN_FILE(`/etc/mail/virtual-host-domains') TRUST_AUTH_MECH(`LOGIN PLAIN') -INPUT_MAIL_FILTER(`dnsbl', `S=local:/var/run/dnsbl/dnsbl.sock, F=T, T=S:30s;R:30s;E:30s') +INPUT_MAIL_FILTER(`dnsbl', `S=local:/var/run/dnsbl/dnsbl.sock2, F=T, T=S:30s;R:30s;E:30s') MAILER(smtp) MAILER(procmail) diff -r 7bb8bbf79285 -r 510a511ad554 xml/dnsbl.in --- a/xml/dnsbl.in Thu Oct 28 22:54:34 2004 -0700 +++ b/xml/dnsbl.in Mon Jan 03 18:35:50 2005 -0800 @@ -2,7 +2,7 @@ -DNSBL Sendmail milter - Version 3.7 +DNSBL Sendmail milter - Version 4.0
Introduction
@@ -31,8 +31,7 @@ startup, and whenever the config file (or any of the referenced include files) is changed. The entire configuration file is case insensitive. -
-
DCC Issues
+
DCC Issues

If you are also using the DCC milter, there are a few considerations. You may need to whitelist senders from the DCC @@ -69,8 +68,7 @@ appropriately tagged and used only for the domains controlled by each of those clients. -

-
Definitions
+
Definitions

DNSBL - a named DNS based blocking list is defined by a dns suffix (e.g. sbl-xbl.spamhaus.org) and a message string that is used to generate the "550 5.7.1" smtp error return code. The names of these @@ -121,8 +119,7 @@ -

-
Sendmail access vs. DNSBL
+
Sendmail access vs. DNSBL

With the standard sendmail.mc dnsbl FEATURE, the dnsbl checks may be suppressed by entries in the /etc/mail/access database. For example, suppose you control a /18 of address space, and have allocated some /24s @@ -152,13 +149,13 @@ these directions helpful for setting up smtp-auth if you are on RH Linux. -

Installation and configuration

Usage: Note -that this has ONLY been tested on Linux, specifically RedHat Linux. In -particular, this milter makes no attempt to understand IPv6. Your -mileage will vary. You will need at a minimum a C++ compiler with a -minimally thread safe STL implementation. The distribution includes a -test.cpp program. If it fails this milter won't work. If it passes, -this milter might work. +

Installation and configuration
+

Usage: Note that this has ONLY been tested on Linux, specifically +RedHat Linux. In particular, this milter makes no attempt to understand +IPv6. Your mileage will vary. You will need at a minimum a C++ +compiler with a minimally thread safe STL implementation. The +distribution includes a test.cpp program. If it fails this milter won't +work. If it passes, this milter might work. Fetch dnsbl.tar.gz and @@ -188,10 +185,31 @@ /usr/sbin/dnsbl -c -

+
 
Performance issues

+
+
Consider a high volume high performance machine running sendmail.
+Each sendmail process can do its own dns resolution.  Typically, such
+dns resolver libraries are not thread safe, and so must be protected by
+some sort of mutex in a threaded environment.  When we add a milter to
+sendmail, we now have a collection of sendmail processes, and a
+collection of milter threads.
 
+
We will be doing a lot of dns lookups per mail message, and at least
+some of those will take many tens of seconds.  If all this dns work is
+serialized inside the milter, we have an upper limit of about 25K mail
+messages per day.  That is clearly not sufficient for many sites.
 
+
Since we want to do parallel dns resolution across those milter
+threads, we add another collection of dns resolver processes.  Each
+sendmail process is talking to a milter thread over a socket, and each
+milter thread is talking to a dns resolver process over another socket.
 
+
Suppose we are processing 20 messages per second, and each message
+requires 20 seconds of dns work.  Then we will have 400 sendmail
+processes, 400 milter threads, and 400 dns resolver processes.  Of
+course that steady state is very unlikely to happen.
+
+
 $Id$