# HG changeset patch
# User carl
# Date 1183849839 25200
# Node ID 8d7c439bb6fa9438488e7ead552f0e2aaa32a9f1
# Parent  c7fc218686f5103a61a6c97a2cd8c8323b012116
add auto whitelisting


diff -r c7fc218686f5 -r 8d7c439bb6fa Makefile.am
--- a/Makefile.am	Sat Jul 07 10:26:31 2007 -0700
+++ b/Makefile.am	Sat Jul 07 16:10:39 2007 -0700
@@ -15,6 +15,8 @@
 chkconfig: dnsbl
 	   /usr/bin/getent passwd dnsbl || /usr/sbin/useradd -r -d /etc/dnsbl -M -c "dnsbl pseudo-user" -s /sbin/nologin dnsbl
 	   mv -f $(sysconfdir)/dnsbl/dnsbl /etc/rc.d/init.d
+	   mkdir $(sysconfdir)/dnsbl/autowhite
+	   chown dnsbl:root $(sysconfdir)/dnsbl/autowhite
 	   /sbin/chkconfig --del dnsbl
 	   /sbin/chkconfig --add dnsbl
 
diff -r c7fc218686f5 -r 8d7c439bb6fa dnsbl.conf
--- a/dnsbl.conf	Sat Jul 07 10:26:31 2007 -0700
+++ b/dnsbl.conf	Sat Jul 07 16:10:39 2007 -0700
@@ -99,5 +99,7 @@
         abuse@  abuse;  # replies to abuse reports use the abuse context
         # dcc_from { include "/var/dcc/whitecommon"; };
     };
+
+    autowhite 90 "my-auto-whitelist";
 };
 
diff -r c7fc218686f5 -r 8d7c439bb6fa dnsbl.spec.in
--- a/dnsbl.spec.in	Sat Jul 07 10:26:31 2007 -0700
+++ b/dnsbl.spec.in	Sat Jul 07 16:10:39 2007 -0700
@@ -97,9 +97,13 @@
 %config(noreplace) %{_sysconfdir}/@PACKAGE@
 /etc/rc.d/init.d/@PACKAGE@
 %dir %attr(0750,@PACKAGE@,root) /var/run/@PACKAGE@
+%dir %attr(0750,@PACKAGE@,root) %{_sysconfdir}/@PACKAGE@/autowhite
 
 
 %changelog
+* Sat Jul 07 2007 Carl Byington 6.01
+- GPL3, auto whitelisting
+
 * Wed Aug 02 2006 Carl Byington 5.20
 - http://www.rpm.org/max-rpm/s1-rpm-inside-scripts.html help with postun
 
diff -r c7fc218686f5 -r 8d7c439bb6fa src/context.cpp
--- a/src/context.cpp	Sat Jul 07 10:26:31 2007 -0700
+++ b/src/context.cpp	Sat Jul 07 16:10:39 2007 -0700
@@ -21,6 +21,7 @@
 
 static char* context_version="$Id$";
 
+char *token_autowhite;
 char *token_black;
 char *token_content;
 char *token_context;
@@ -63,10 +64,16 @@
 #endif
 char myhostname[HOST_NAME_MAX+1];
 
+pthread_mutex_t verifier_mutex; 	// protect the verifier map
 verify_map	verifiers;
+
+pthread_mutex_t whitelister_mutex;	// protect the
+whitelister_map whitelisters;
+
 string_set	all_strings;	// owns all the strings, only modified by the config loader thread
 const int maxlen = 1000;	// used for snprintf buffers
-const int maxage = 120; 	// smtp verify sockets older than this are ancient
+const int maxsmtp_age = 120;// smtp verify sockets older than this are ancient
+const int maxauto_age = 600;// auto whitelister delay before flushing to file
 extern int	  NULL_SOCKET;
 const time_t  ERROR_SMTP_SOCKET_TIME = 600; // number of seconds between attempts to open a socket to an smtp server
 
@@ -243,6 +250,9 @@
 #endif
 
 
+////////////////////////////////////////////////
+// smtp verifier so backup mx machines can see the valid users
+//
 VERIFY::VERIFY(char *h) {
 	host	 = h;
 	last_err = 0;
@@ -261,7 +271,7 @@
 			else {
 				conn = connections.front();
 				time_t now = time(NULL);
-				if ((now - conn->get_stamp()) > maxage) {
+				if ((now - conn->get_stamp()) > maxsmtp_age) {
 					// this connection is ancient, remove it
 					connections.pop_front();
 				}
@@ -372,6 +382,140 @@
 }
 
 
+////////////////////////////////////////////////
+// setup a new smtp verify host
+//
+VERIFYP add_verify_host(char *host);
+VERIFYP add_verify_host(char *host) {
+	VERIFYP rc = NULL;
+	pthread_mutex_lock(&verifier_mutex);
+		verify_map::iterator i = verifiers.find(host);
+		if (i == verifiers.end()) {
+			rc = new VERIFY(host);
+			verifiers[host] = rc;
+		}
+		else rc = (*i).second;
+	pthread_mutex_unlock(&verifier_mutex);
+	return rc;
+}
+
+
+////////////////////////////////////////////////
+// thread to check for verify hosts with old sockets that we can close
+//
+void* verify_closer(void *arg) {
+	while (true) {
+		sleep(maxsmtp_age);
+		pthread_mutex_lock(&verifier_mutex);
+			for (verify_map::iterator i=verifiers.begin(); i!=verifiers.end(); i++) {
+				VERIFYP v = (*i).second;
+				v->closer();
+			}
+		pthread_mutex_unlock(&verifier_mutex);
+	}
+	return NULL;
+}
+
+
+////////////////////////////////////////////////
+// automatic whitelister
+//
+WHITELISTER::WHITELISTER(char *f, int d) {
+	fn		 = f;
+	days	 = d;
+	pthread_mutex_init(&mutex, 0);
+	need	 = false;
+}
+
+
+void WHITELISTER::writer() {
+	pthread_mutex_lock(&mutex);
+		time_t limit = time(NULL) - days*86400;
+		for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end();) {
+			time_t when = (*i).second;
+			if (when < limit) {
+				autowhite_sent::iterator j = i;
+				j++;
+				rcpts.erase(i);
+				i = j;
+				need = true;
+			}
+			else i++;
+		}
+		if (need) {
+			// dump the file
+			ofstream os;
+			os.open(fn);
+			if (!os.fail()) {
+				for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end(); i++) {
+					char *who = (*i).first;
+					int  when = (*i).second;
+					os << who << " " << when << endl;
+				}
+			}
+			os.close();
+		}
+	pthread_mutex_unlock(&mutex);
+}
+
+
+void WHITELISTER::sent(char *to) {
+	pthread_mutex_lock(&mutex);
+		need = true;
+		rcpts[to] = time(NULL);
+	pthread_mutex_unlock(&mutex);
+}
+
+
+bool WHITELISTER::is_white(char *from) {
+	bool rc = false;
+	pthread_mutex_lock(&mutex);
+		autowhite_sent::iterator i = rcpts.find(from);
+		if (i != rcpts.end()) {
+			time_t when = (*i).second;
+			time_t now = time(NULL);
+			rc = (when+(days*8640) > now);
+		}
+	pthread_mutex_unlock(&mutex);
+	return rc;
+}
+
+
+////////////////////////////////////////////////
+// setup a new auto whitelister file
+//
+WHITELISTERP add_whitelister_file(char *fn, int days);
+WHITELISTERP add_whitelister_file(char *fn, int days) {
+	WHITELISTERP rc = NULL;
+	pthread_mutex_lock(&whitelister_mutex);
+		whitelister_map::iterator i = whitelisters.find(fn);
+		if (i == whitelisters.end()) {
+			rc = new WHITELISTER(fn, days);
+			whitelisters[fn] = rc;
+		}
+		else rc = (*i).second;
+	pthread_mutex_unlock(&whitelister_mutex);
+	return rc;
+}
+
+
+////////////////////////////////////////////////
+// thread to check for whitelister hosts with old sockets that we can close
+//
+void* whitelister_writer(void *arg) {
+	while (true) {
+		sleep(maxauto_age);
+		pthread_mutex_lock(&whitelister_mutex);
+			for (whitelister_map::iterator i=whitelisters.begin(); i!=whitelisters.end(); i++) {
+				WHITELISTERP v = (*i).second;
+				v->writer();
+			}
+		pthread_mutex_unlock(&whitelister_mutex);
+	}
+	return NULL;
+}
+
+
 DNSBL::DNSBL(char *n, char *s, char *m) {
 	name	= n;
 	suffix	= s;
@@ -474,6 +618,9 @@
 	parent				= parent_;
 	name				= name_;
 	verify_host 		= NULL;
+	verifier			= NULL;
+	autowhite_file		= NULL;
+	whitelister 		= NULL;
 	env_from_default	= (parent) ? token_inherit : token_unknown;
 	content_filtering	= (parent) ? parent->content_filtering : false;
 	content_suffix		= NULL;
@@ -531,22 +678,22 @@
 
 
 VERIFYP CONTEXT::find_verify(char *to) {
-	if (verify_host && (verify_host != token_myhostname) && cover_env_to(to)) {
-		verify_map::iterator i = verifiers.find(verify_host);
-		if (i == verifiers.end()) {
-			if (debug_syslog) {
-				char buf[maxlen];
-				snprintf(buf, maxlen, "cannot find struc for %s", verify_host);
-				my_syslog(buf);
-			}
+	if (verifier && (verify_host != token_myhostname) && cover_env_to(to))
+		return verifier;
+	else if (parent)
+		return parent->find_verify(to);
+	else
 			return NULL;
 		}
-		VERIFYP v = (*i).second;
+
 
-		return v;
-	}
-	else if (parent) return parent->find_verify(to);
-	else return NULL;
+WHITELISTERP CONTEXT::find_autowhite(char *to) {
+	if (whitelister && cover_env_to(to))
+		return whitelister;
+	else if (parent)
+		return parent->find_autowhite(to);
+	else
+		return NULL;
 }
 
 
@@ -558,6 +705,7 @@
 
 
 char *CONTEXT::find_from(char *from) {
+	if (whitelister && whitelister->is_white(from)) return token_white;
 	char *rc = env_from_default;
 	string_map::iterator i = env_from.find(from);
 	if (i != env_from.end()) rc = (*i).second;	// found user@domain key
@@ -775,6 +923,10 @@
 		printf("%s     verify %s; \n", indent, verify_host);
 	}
 
+	if (autowhite_file && whitelister) {
+		printf("%s     autowhite %d %s; \n", indent, whitelister->get_days(), autowhite_file);
+	}
+
 	for (context_map::iterator i=children.begin(); i!=children.end(); i++) {
 		CONTEXTP c = (*i).second;
 		c->dump(false, level+1);
@@ -1100,7 +1252,20 @@
 	char *host = tok.next();
 	if (!tsa(tok, token_semi)) return false;
 	me.set_verify(host);
-	add_verify_host(host);
+	me.set_verifier(add_verify_host(host));
+	return true;
+}
+
+
+////////////////////////////////////////////////
+//
+bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me);
+bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
+	int days = tok.nextint();
+	char *fn = tok.next();
+	if (!tsa(tok, token_semi)) return false;
+	me.set_autowhite(fn);
+	me.set_whitelister(add_whitelister_file(fn, days));
 	return true;
 }
 
@@ -1233,6 +1398,9 @@
 		else if (have == token_verify) {
 			if (!parse_verify(tok, dc, *con)) return false;
 		}
+		else if (have == token_autowhite) {
+			if (!parse_autowhite(tok, dc, *con)) return false;
+		}
 		else if (have == token_envfrom) {
 			if (!parse_envfrom(tok, dc, *con)) return false;
 		}
@@ -1286,36 +1454,10 @@
 
 
 ////////////////////////////////////////////////
-// setup a new smtp verify host
-//
-void add_verify_host(char *host) {
-	verify_map::iterator i = verifiers.find(host);
-	if (i == verifiers.end()) {
-		VERIFYP v = new VERIFY(host);
-		verifiers[host] = v;
-	}
-}
-
-
-////////////////////////////////////////////////
-// thread to check for verify hosts with old sockets that we can close
-//
-void* verify_closer(void *arg) {
-	while (true) {
-		sleep(maxage);
-		for (verify_map::iterator i=verifiers.begin(); i!=verifiers.end(); i++) {
-			VERIFYP v = (*i).second;
-			v->closer();
-		}
-	}
-	return NULL;
-}
-
-
-////////////////////////////////////////////////
 // init the tokens
 //
 void token_init() {
+	token_autowhite  = register_string("autowhite");
 	token_black 	 = register_string("black");
 	token_cctld 	 = register_string("cctld");
 	token_content	 = register_string("content");
diff -r c7fc218686f5 -r 8d7c439bb6fa src/context.h
--- a/src/context.h	Sat Jul 07 10:26:31 2007 -0700
+++ b/src/context.h	Sat Jul 07 16:10:39 2007 -0700
@@ -22,6 +22,7 @@
 class CONTEXT;
 class VERIFY;
 class SMTP;
+class WHITELISTER;
 class recorder;
 
 typedef map<char *, char *, ltstr>		  string_map;
@@ -30,6 +31,7 @@
 typedef list<char *>					  string_list;
 typedef DNSBL * 						  DNSBLP;
 typedef VERIFY *						  VERIFYP;
+typedef WHITELISTER *					  WHITELISTERP;
 typedef list<DNSBLP>					  dnsblp_list;
 typedef map<char *, DNSBLP, ltstr>		  dnsblp_map;
 typedef CONTEXT *						  CONTEXTP;
@@ -37,7 +39,9 @@
 typedef map<char *, CONTEXTP, ltstr>	  context_map;
 typedef map<char *, int, ltstr> 		  ns_mapper;
 typedef map<char *, int, ltstr> 		  rcpt_rates;
+typedef map<char *, int, ltstr> 		  autowhite_sent;
 typedef map<char *, VERIFYP, ltstr> 	  verify_map;
+typedef map<char *, WHITELISTERP, ltstr>  whitelister_map;
 
 class SMTP {
 	static const int maxlen = 1000;
@@ -87,6 +91,20 @@
 	bool	ok(char *from, char *to);
 };
 
+class WHITELISTER {
+	char			*fn;		// file to use
+	int 			days;		// how long do we keep entries
+	pthread_mutex_t mutex;		// protect the flag and map
+	bool			need;		// force writing on new entries
+	autowhite_sent	rcpts;		// recipient map to remember when we sent them mail
+public:
+	WHITELISTER(char *f, int d);
+	void	writer();			// dump any changes back to the file
+	void	sent(char *to);
+	bool	is_white(char *from);	// should we white list this sender (did we send them anything recently)
+	int 	get_days() {return days;};
+};
+
 struct DNSBL {
 	char	*name;		// nickname for this dns based list
 	char	*suffix;	// blacklist suffix like blackholes.five-ten-sg.com
@@ -101,6 +119,9 @@
 	context_map 	children;			// map child context names to their contexts
 	string_set		env_to; 			// this context applies to these envelope recipients
 	char *			verify_host;		// use this smtp host to verify email addresses
+	VERIFYP 		verifier;			// pointer to the verifier structure
+	char *			autowhite_file; 	// file to use for automatic whitelisting
+	WHITELISTERP	whitelister;		// pointer to the auto whitelister structure
 	string_map		env_from;			// map senders to white/black/unknown
 	context_map 	env_from_context;	// map senders to a child context
 	char *			env_from_default;	// default value for senders that are not found in the map white/black/unknown/inherit
@@ -134,10 +155,16 @@
 	bool		allow_env_to(char *to)						{return (parent) ? parent->cover_env_to(to) : true;};
 	bool		cover_env_to(char *to);
 
+	void		set_verifier(VERIFYP v) 					{verifier	 = v;};
 	void		set_verify(char *host)						{verify_host = host;};
 	char*		get_verify()								{return verify_host;};
 	VERIFYP 	find_verify(char *to);
 
+	void		set_whitelister(WHITELISTERP v) 			{whitelister	= v;};
+	void		set_autowhite(char *fn) 					{autowhite_file = fn;};
+	char*		get_autowhite() 							{return autowhite_file;};
+	WHITELISTERP find_autowhite(char *to);
+
 	void		set_default_rate(int limit) 				{default_rcpt_rate	 = limit;};
 	void		add_rate(char *user, int limit) 			{rcpt_per_hour[user] = limit;};
 	int 		find_rate(char *user);
@@ -214,6 +241,7 @@
 
 };
 
+extern char *token_autowhite;
 extern char *token_black;
 extern char *token_cctld;
 extern char *token_content;
@@ -249,18 +277,16 @@
 extern char *token_uribl;
 extern char *token_white;
 
-extern char *token_myhostname;
-
-extern verify_map	verifiers;		// map of smtp hosts to verify structures, owns all the verify structures
-extern string_set	all_strings;	// owns all the strings, only modified by the config loader thread
+extern pthread_mutex_t verifier_mutex;	   // protect the verifier map
+extern pthread_mutex_t whitelister_mutex;  // protect the
 
 void discard(string_set &s);
 char* register_string(string_set &s, char *name);
 char* register_string(char *name);
 CONFIG *parse_config(char *fn);
 bool  load_conf(CONFIG &dc, char *fn);
-void  add_verify_host(char *host);
 void* verify_closer(void *arg);
+void* whitelister_writer(void *arg);
 void  token_init();
 
 #endif
diff -r c7fc218686f5 -r 8d7c439bb6fa src/dnsbl.cpp
--- a/src/dnsbl.cpp	Sat Jul 07 10:26:31 2007 -0700
+++ b/src/dnsbl.cpp	Sat Jul 07 16:10:39 2007 -0700
@@ -1021,6 +1021,14 @@
 			return SMFIS_REJECT;
 		}
 	}
+	// we will accept the recipient, but add an auto-whitelist entry
+	// if needed to ensure we can accept replies
+	WHITELISTERP w = con2.find_autowhite(priv.mailaddr);
+	if (w) {
+		char *loto = to_lower_string(rcptaddr);
+		w->sent(loto);
+		free(loto);
+	}
 	// accept the recipient
 	if (!con.get_content_filtering()) st = white;
 	if (st == oksofar) {
@@ -1449,6 +1457,8 @@
 	pthread_mutex_init(&syslog_mutex, 0);
 	pthread_mutex_init(&resolve_mutex, 0);
 	pthread_mutex_init(&fd_pool_mutex, 0);
+	pthread_mutex_init(&verifier_mutex, 0);
+	pthread_mutex_init(&whitelister_mutex, 0);
 
 	// drop root privs
 	struct passwd *pw = getpwnam("dnsbl");
@@ -1549,6 +1559,11 @@
 	if (pthread_detach(tid))
 		my_syslog("failed to detach verify closer thread");
 
+	if (pthread_create(&tid, 0, whitelister_writer, 0))
+		my_syslog("failed to create autowhite writer thread");
+	if (pthread_detach(tid))
+		my_syslog("failed to detach autowhite writer thread");
+
 	time_t starting = time(NULL);
 	int rc = smfi_main();
 	if ((rc != MI_SUCCESS) && (time(NULL) > starting+5*60)) {
diff -r c7fc218686f5 -r 8d7c439bb6fa xml/dnsbl.in
--- a/xml/dnsbl.in	Sat Jul 07 10:26:31 2007 -0700
+++ b/xml/dnsbl.in	Sat Jul 07 16:10:39 2007 -0700
@@ -538,7 +538,7 @@
 CONFIG     = {CONTEXT ";"}+
 CONTEXT    = "context" NAME "{" {STATEMENT}+ "}"
 STATEMENT  = (DNSBL | DNSBLLIST | CONTENT | ENV-TO | VERIFY |
-                             CONTEXT | ENV-FROM | RATE-LIMIT) ";"
+                      AUTOWHITE | CONTEXT | ENV-FROM | RATE-LIMIT) ";"
 
 DNSBL      = "dnsbl" NAME DNSPREFIX ERROR-MSG1
 
@@ -571,6 +571,7 @@
 DCC-TO     = "dcc_to" ("ok" | "many") "{" DCCINCLUDEFILE "}" ";"
 
 VERIFY     = "verify" HOSTNAME ";"
+AUTOWHITE  = "autowhite" DAYS FILENAME ";"
 
 ENV_FROM   = "env_from" [DEFAULT] "{" {(FROM-ADDR | DCC-FROM)}+ "}"
 FROM-ADDR  = ADDRESS VALUE [";"]
@@ -701,8 +702,12 @@
             customer1b.com;
         };
 
+        # we can reject unknown users
         verify mail.customer1.com;
 
+        # whitelist anyone to whom we have sent mail in the last 90 days
+        autowhite 90 "autowhite/customer1";
+
         context customer1a {
             env_to {
                 customer1a.com;