# HG changeset patch # User carl # Date 1127314808 25200 # Node ID 505e771883179957f50636d347da426c43f8ee5d # Parent ca46fafc66216fdb119746f68a248e66feac9f60 optimize verification step, cleanup documentation diff -r ca46fafc6621 -r 505e77188317 ChangeLog --- a/ChangeLog Sun Sep 18 10:46:31 2005 -0700 +++ b/ChangeLog Wed Sep 21 08:00:08 2005 -0700 @@ -1,5 +1,10 @@ $Id$ +5.5 2005-09-21 + Cleanup debug logging. Verify from/to pairs now remembers the last + from value sent to the remote server to prevent unnecessary rset + commands. + 5.4 2005-09-18 Add verify statement to specify the smtp host to be used to verify envelope from / recipient pairs. diff -r ca46fafc6621 -r 505e77188317 dnsbl.conf --- a/dnsbl.conf Sun Sep 18 10:46:31 2005 -0700 +++ b/dnsbl.conf Wed Sep 21 08:00:08 2005 -0700 @@ -2,7 +2,8 @@ dnsbl local blackholes.five-ten-sg.com "Mail from %s rejected - local; see http://www.five-ten-sg.com/blackhole.php?%s"; dnsbl sbl sbl-xbl.spamhaus.org "Mail from %s rejected - sbl; see http://www.spamhaus.org/query/bl?ip=%s"; dnsbl xbl xbl.spamhaus.org "Mail from %s rejected - xbl; see http://www.spamhaus.org/query/bl?ip=%s"; - dnsbl_list local sbl; + dnsbl dul dul.dnsbl.sorbs.net "Mail from %s rejected - dul; see http://www.sorbs.net/lookup.shtml?%s"; + dnsbl_list local sbl dul; content on { filter sbl-xbl.spamhaus.org "Mail containing %s rejected - sbl; see http://www.spamhaus.org/query/bl?ip=%s"; @@ -38,7 +39,7 @@ }; context minimal { - dnsbl_list sbl; + dnsbl_list sbl dul; content on {}; env_to { }; diff -r ca46fafc6621 -r 505e77188317 dnsbl.spec.in --- a/dnsbl.spec.in Sun Sep 18 10:46:31 2005 -0700 +++ b/dnsbl.spec.in Wed Sep 21 08:00:08 2005 -0700 @@ -1,6 +1,6 @@ Summary: DNSBL Sendmail Milter Name: dnsbl -Version: 5.4 +Version: 5.5 Release: 2 Copyright: GPL Group: System Environment/Daemons diff -r ca46fafc6621 -r 505e77188317 install.bash --- a/install.bash Sun Sep 18 10:46:31 2005 -0700 +++ b/install.bash Wed Sep 21 08:00:08 2005 -0700 @@ -62,6 +62,7 @@ if [ ! -f $DST/html-tags.conf ]; then cp html-tags.conf $DST fi +rm -f $DST/tld.conf # new tld list if [ ! -f $DST/tld.conf ]; then cp tld.conf $DST fi diff -r ca46fafc6621 -r 505e77188317 package.bash --- a/package.bash Sun Sep 18 10:46:31 2005 -0700 +++ b/package.bash Wed Sep 21 08:00:08 2005 -0700 @@ -1,6 +1,6 @@ #!/bin/bash -VER=dnsbl-5.4 +VER=dnsbl-5.5 mkdir $VER target1=/home/httpd/html/510sg/util/dnsbl.tar.gz target2=/home/httpd/html/510sg/dnsbl.conf diff -r ca46fafc6621 -r 505e77188317 sendmail.st Binary file sendmail.st has changed diff -r ca46fafc6621 -r 505e77188317 src/context.cpp --- a/src/context.cpp Sun Sep 18 10:46:31 2005 -0700 +++ b/src/context.cpp Wed Sep 21 08:00:08 2005 -0700 @@ -65,8 +65,10 @@ int SMTP::writer() { - // log("writer() sees buffer with %s", buffer); - // log("writer() sees error %d", (int)error); + #ifdef VERIFY_DEBUG + log("writer() sees buffer with %s", buffer); + log("writer() sees error %d", (int)error); + #endif int rs = 0; if (!error) { int len = strlen(buffer); @@ -90,7 +92,9 @@ int SMTP::reader() { // read some bytes terminated by lf or end of buffer. // we may have a multi line response or part thereof in the buffer. - // log("reader() sees error %d", (int)error); + #ifdef VERIFY_DEBUG + log("reader() sees error %d", (int)error); + #endif if (error) return 0; int len = maxlen-1; // room for null terminator while (pending < len) { @@ -107,7 +111,9 @@ } } buffer[pending] = '\0'; - // log("reader() sees buffer with %s", buffer); + #ifdef VERIFY_DEBUG + log("reader() sees buffer with %s", buffer); + #endif return pending; } @@ -134,8 +140,10 @@ buffer[pending] = '\0'; while (true) { int r = read_line(); - // log("read_response() sees line with %s", buffer); - // log("read_response() sees line length %d", r); + #ifdef VERIFY_DEBUG + log("read_response() sees line with %s", buffer); + log("read_response() sees line length %d", r); + #endif if (r == 0) return 0; // failed to read any bytes if ((r > 4) && (buffer[3] == '-')) { flush_line(r); @@ -168,17 +176,24 @@ int SMTP::rset() { - return cmd("RSET"); + int rc = cmd("RSET"); + efrom[0] = '\0'; + return rc; } int SMTP::from(char *f) { + if (strncmp(efrom, f, maxlen)) { + rset(); + strncpy(efrom, f, maxlen); init(); append("MAIL FROM:<"); append(f); append(">"); return cmd(NULL); } + return 250; // pretend it worked +} int SMTP::rcpt(char *t) { @@ -191,27 +206,32 @@ int SMTP::quit() { - int rc = cmd("QUIT"); + return cmd("QUIT"); +} + + +void SMTP::closefd() { shutdown(fd, SHUT_RDWR); close(fd); - return rc; } -// void SMTP::log(char *m, int v) { -// char buf[maxlen]; -// snprintf(buf, maxlen, m, v); -// my_syslog(buf); -// } -// -// -// void SMTP::log(char *m, char *v) { -// char buf[maxlen]; -// snprintf(buf, maxlen, m, v); -// my_syslog(buf); -// } -// -// +#ifdef VERIFY_DEBUG + void SMTP::log(char *m, int v) { + char buf[maxlen]; + snprintf(buf, maxlen, m, v); + my_syslog(buf); + } + + + void SMTP::log(char *m, char *v) { + char buf[maxlen]; + snprintf(buf, maxlen, m, v); + my_syslog(buf); + } +#endif + + VERIFY::VERIFY(char *h) { host = h; last_err = 0; @@ -222,47 +242,49 @@ void VERIFY::closer() { bool ok = true; while (ok) { - int fd = 0; + SMTP *conn = NULL; pthread_mutex_lock(&mutex); - if (sockets.empty()) { + if (connections.empty()) { ok = false; } else { - time_t t = times.front(); + conn = connections.front(); time_t now = time(NULL); - if ((now - t) > maxage) { - // this socket is ancient, remove it - fd = sockets.front(); - times.pop_front(); - sockets.pop_front(); + if ((now - conn->get_stamp()) > maxage) { + // this connection is ancient, remove it + connections.pop_front(); } else { ok = false; + conn = NULL; } } pthread_mutex_unlock(&mutex); - if (fd) { - SMTP s(fd); - s.quit(); // closes the fd - // s.log("closer() closes ancient %d", fd); + // avoid doing this work inside the mutex lock + if (conn) { + #ifdef VERIFY_DEBUG + conn->log("closer() closes ancient %d", conn->get_fd()); + #endif + delete conn; } } } -int VERIFY::get_socket() { - int sock = NULL_SOCKET; +SMTP* VERIFY::get_connection() { + SMTP *conn = NULL; pthread_mutex_lock(&mutex); - if (!sockets.empty()) { - sock = sockets.front(); - times.pop_front(); - sockets.pop_front(); - // SMTP::log("get_socket() %d from cache", sock); + if (!connections.empty()) { + conn = connections.front(); + connections.pop_front(); + #ifdef VERIFY_DEBUG + conn->log("get_connection() %d from cache", conn->get_fd()); + #endif } pthread_mutex_unlock(&mutex); - - if (sock == NULL_SOCKET) { + if (conn) return conn; time_t now = time(NULL); + int sock = NULL_SOCKET; if ((now - last_err) > ERROR_SOCKET_TIME) { // nothing recent, maybe this time it will work hostent *h = gethostbyname(host); @@ -286,30 +308,32 @@ else last_err = now; } if (sock != NULL_SOCKET) { - SMTP s(sock); - if (s.helo() != 250) { - put_socket(sock, true); - sock = NULL_SOCKET; + conn = new SMTP(sock); + #ifdef VERIFY_DEBUG + conn->log("get_connection() %d new socket", conn->get_fd()); + #endif + if (conn->helo() == 250) return conn; + delete conn; } - } - } - return sock; + return NULL; } -void VERIFY::put_socket(int fd, bool err) { - if (err) { - // SMTP::log("put_socket() %d with error, close it", fd); - shutdown(fd, SHUT_RDWR); - close(fd); +void VERIFY::put_connection(SMTP *conn) { + if (conn->err()) { + #ifdef VERIFY_DEBUG + conn->log("put_socket() %d with error, close it", conn->get_fd()); + #endif + delete conn; last_err = time(NULL); } else { - // SMTP::log("put_socket() %d", fd); + #ifdef VERIFY_DEBUG + conn->log("put_socket() %d", conn->get_fd()); + #endif + conn->now(); pthread_mutex_lock(&mutex); - time_t now = time(NULL); - times.push_back(now); - sockets.push_back(fd); + connections.push_back(conn); pthread_mutex_unlock(&mutex); } } @@ -317,20 +341,23 @@ bool VERIFY::ok(char *from, char *to) { if (host == token_myhostname) return true; - int fd = get_socket(); - if (fd == NULL_SOCKET) return true; // cannot verify right now, we have socket errors - SMTP s(fd); - s.rset(); + SMTP *conn = get_connection(); + if (!conn) return true; // cannot verify right now, we have socket errors int rc; - rc = s.from(from); - // s.log("verify::ok from sees %d", rc); + rc = conn->from(from); + #ifdef VERIFY_DEBUG + conn->log("verify::ok() from sees %d", rc); + #endif if (rc != 250) { - put_socket(fd, s.err()); + conn->rset(); + put_connection(conn); return (rc >= 500) ? false : true; } - rc = s.rcpt(to); - // s.log("verify::ok rcpt sees %d", rc); - put_socket(fd, s.err()); + rc = conn->rcpt(to); + #ifdef VERIFY_DEBUG + conn->log("verify::ok() rcpt sees %d", rc); + #endif + put_connection(conn); return (rc >= 500) ? false : true; } diff -r ca46fafc6621 -r 505e77188317 src/context.h --- a/src/context.h Sun Sep 18 10:46:31 2005 -0700 +++ b/src/context.h Wed Sep 21 08:00:08 2005 -0700 @@ -13,12 +13,12 @@ class DNSBL; class CONTEXT; class VERIFY; +class SMTP; class recorder; typedef map string_map; typedef set int_set; -typedef list fd_list; -typedef list time_list; +typedef list smtp_list; typedef list string_list; typedef DNSBL * DNSBLP; typedef VERIFY * VERIFYP; @@ -34,13 +34,19 @@ static const int maxlen = 1000; int fd; bool error; + time_t stamp; + char efrom[maxlen]; // last envelope from sent on this socket int pending; // unread bytes in buffer, not including the null terminator char buffer[maxlen]; public: - SMTP(int f) {fd = f; error = false;}; + SMTP(int f) {fd = f; error = false; efrom[0] = '\0';}; + ~SMTP() {if (!error) quit(); closefd();}; void init() {pending = 0; buffer[0] = '\0';}; void append(char *c) {strncat(buffer, c, max(0, maxlen-1-(int)strlen(c)));}; bool err() {return error;}; + void now() {stamp = time(NULL);}; + time_t get_stamp() {return stamp;}; + int get_fd() {return fd;}; int writer(); int reader(); int read_line(); @@ -52,21 +58,23 @@ int from(char *f); int rcpt(char *t); int quit(); - // static void log(char *m, int v); - // static void log(char *m, char *v); + void closefd(); +#ifdef VERIFY_DEBUG + static void log(char *m, int v); + static void log(char *m, char *v); +#endif }; class VERIFY { char *host; // host to be used to verify recipient addresses time_t last_err; // time of last socket error pthread_mutex_t mutex; // protect the lists of sockets and timestamps - fd_list sockets; // open sockets, ready to be used - time_list times; // last timestamp when this socket was used + smtp_list connections;// open sockets, ready to be used public: VERIFY(char *h); void closer(); // if the oldest socket is ancient, close it - int get_socket(); - void put_socket(int fd, bool err); + SMTP *get_connection(); + void put_connection(SMTP *conn); bool ok(char *from, char *to); }; diff -r ca46fafc6621 -r 505e77188317 src/dnsbl.cpp --- a/src/dnsbl.cpp Sun Sep 18 10:46:31 2005 -0700 +++ b/src/dnsbl.cpp Wed Sep 21 08:00:08 2005 -0700 @@ -392,8 +392,6 @@ syslog_opened = true; } syslog(LOG_NOTICE, "%s", text); - // closelog(); - // syslog_opened = false; pthread_mutex_unlock(&syslog_mutex); } else { @@ -434,7 +432,9 @@ } else { // peer closed the socket - //my_syslog("!!child worker process, peer closed socket while reading question"); + #ifdef RESOLVER_DEBUG + my_syslog("process_resolver_requests() peer closed socket while reading question"); + #endif shutdown(socket, SHUT_RDWR); close(socket); return; @@ -444,9 +444,11 @@ // find the answer #ifdef NS_PACKETSZ - //char text[1000]; - //snprintf(text, sizeof(text), "!!child worker process has a question %s", question); - //my_syslog(text); + #ifdef RESOLVER_DEBUG + char text[1000]; + snprintf(text, sizeof(text), "process_resolver_requests() has a question %s", question); + my_syslog(text); + #endif glom.length = res_search(question, ns_c_in, ns_t_a, glom.answer, sizeof(glom.answer)); if (glom.length < 0) glom.length = 0; // represent all errors as zero length answers #else @@ -461,8 +463,10 @@ // write the answer char *buf = (char *)&glom; int len = glom.length + sizeof(glom.length); - //snprintf(text, sizeof(text), "!!child worker process writing answer length %d for total %d", glom.length, len); - //my_syslog(text); + #ifdef RESOLVER_DEBUG + snprintf(text, sizeof(text), "process_resolver_requests() writing answer length %d for total %d", glom.length, len); + my_syslog(text); + #endif int ws = 0; while (len > ws) { int ns = write(socket, buf+ws, len-ws); @@ -471,7 +475,9 @@ } else { // peer closed the socket! - //my_syslog("!!child worker process, peer closed socket while writing answer"); + #ifdef RESOLVER_DEBUG + my_syslog("process_resolver_requests() peer closed socket while writing answer"); + #endif shutdown(socket, SHUT_RDWR); close(socket); return; @@ -498,9 +504,11 @@ char *buf = (char *)&glom; priv.my_read(buf, sizeof(glom.length)); buf += sizeof(glom.length); - ///char text[1000]; - ///snprintf(text, sizeof(text), "!!milter thread wrote question %s and has answer length %d", question, glom.length); - ///my_syslog(text); + #ifdef RESOLVER_DEBUG + char text[1000]; + snprintf(text, sizeof(text), "dns_interface() wrote question %s and has answer length %d", question, glom.length); + my_syslog(text); + #endif if ((glom.length < 0) || (glom.length > sizeof(glom.answer))) { priv.err = true; return 0; // cannot process overlarge answers diff -r ca46fafc6621 -r 505e77188317 src/includes.h --- a/src/includes.h Sun Sep 18 10:46:31 2005 -0700 +++ b/src/includes.h Wed Sep 21 08:00:08 2005 -0700 @@ -1,3 +1,8 @@ +#define VERIFY_DEBUG 1 +#define RESOLVER_DEBUG 1 +#undef VERIFY_DEBUG +#undef RESOLVER_DEBUG + #include "tokenizer.h" #include "context.h" #include "dnsbl.h" diff -r ca46fafc6621 -r 505e77188317 tld.conf --- a/tld.conf Sun Sep 18 10:46:31 2005 -0700 +++ b/tld.conf Wed Sep 21 08:00:08 2005 -0700 @@ -1,5 +1,5 @@ # -# icann tlds from http://www.icann.org/registries/listing.html +# icann tlds from http://www.icann.org/registries/listing.html as of 2005-09-18 # aero biz @@ -9,6 +9,7 @@ gov info int +jobs mil museum name @@ -16,7 +17,7 @@ org pro # -# icann tlds from http://www.iana.org/cctld/cctld-whois.htm 2004-06-04 +# icann tlds from http://www.iana.org/cctld/cctld-whois.htm as of 2004-06-04 # ac # Ascension Island ad # Andorra diff -r ca46fafc6621 -r 505e77188317 xml/dnsbl.in --- a/xml/dnsbl.in Sun Sep 18 10:46:31 2005 -0700 +++ b/xml/dnsbl.in Wed Sep 21 08:00:08 2005 -0700 @@ -2,7 +2,7 @@ -DNSBL Sendmail milter - Version 5.4 +DNSBL Sendmail milter - Version 5.5
Introduction
@@ -19,6 +19,13 @@ per-recipient basis, so that fred@example.com could use SPEWS and the SBL, where all other users @example.com use only the SBL. +

This milter can also verify the envelope from/recipient pairs with +the primary MX server. This allows the backup mail servers to properly +reject mail sent to invalid addresses. Otherwise, the backup mail +servers will accept that mail, and then generate a bounce message when +the message is forwarded to the primary server (and rejected there with +no such user). +

This milter will also decode (uuencode, base64, mime, html entity, url encodings) and scan for HTTP and HTTPS URLs and bare hostnames in the body of the mail. If any of those host names have A or NS records @@ -112,14 +119,6 @@ point to a child context. If such an entry is found, we switch to that child filtering context. -

  • If the filtering context specifies a verification host, and -the envelope to email address is covered by this filtering context, and -the verification host is not our own hostname, we open an smtp -conversation with that verification host. The current envelope from and -recipient to values are passed to that verification host. If we receive -anything other than a 250 response those commands, we reject the current -recipient with "no such user". -

  • We lookup [1) the full envelope from email address, 2) the domain name part of the envelope from address, 3) the user@ part of the envelope from address] in the filtering context env_from statement. @@ -143,9 +142,18 @@ dns suffix).

  • If the mail has not been accepted or rejected yet, and the +filtering context specifies a verification host, and the envelope to +email address is covered by this filtering context, and the verification +host is not our own hostname, we open an smtp conversation with that +verification host. The current envelope from and recipient to values +are passed to that verification host. If we receive a 5xy response +those commands, we reject the current recipient with "no such user". + +

  • If the mail has not been accepted or rejected yet, and the filtering context enables content filtering, and this is the first such -recipient in this smtp transaction, we set the content filtering parameters -from this context, and enable content filtering for this body. +recipient in this smtp transaction, we set the content filtering +parameters from this context, and enable content filtering for the body +of this message. diff -r ca46fafc6621 -r 505e77188317 xml/sample.conf --- a/xml/sample.conf Sun Sep 18 10:46:31 2005 -0700 +++ b/xml/sample.conf Wed Sep 21 08:00:08 2005 -0700 @@ -40,10 +40,10 @@ context sample { dnsbl local blackholes.five-ten-sg.com "Mail from %s rejected - local; see http://www.five-ten-sg.com/blackhole.php?%s"; - dnsbl spews blackholes.spews.org "Mail from %s rejected - spews; see http://www.spews.org/ask.cgi?x=%s"; dnsbl sbl sbl-xbl.spamhaus.org "Mail from %s rejected - sbl; see http://www.spamhaus.org/query/bl?ip=%s"; dnsbl xbl xbl.spamhaus.org "Mail from %s rejected - xbl; see http://www.spamhaus.org/query/bl?ip=%s"; - dnsbl_list local sbl; + dnsbl dul dul.dnsbl.sorbs.net "Mail from %s rejected - dul; see http://www.sorbs.net/lookup.shtml?%s"; + dnsbl_list local sbl dul; content on { filter sbl-xbl.spamhaus.org "Mail containing %s rejected - sbl; see http://www.spamhaus.org/query/bl?ip=%s"; @@ -86,7 +86,7 @@ }; context minimal { - dnsbl_list sbl; + dnsbl_list sbl dul; content on {}; env_to { sales@mydomain.com; @@ -114,7 +114,7 @@ }; context customer1 { - dnsbl_list sbl; + dnsbl_list sbl dul; env_to { customer1.com; customer1a.com; @@ -139,7 +139,7 @@ }; context customer2 { - dnsbl_list sbl spews; + dnsbl_list sbl; env_to { customer2.com; customer2a.com;