/*

Copyright (c) 2007 Carl Byington - 510 Software Group, released under
the GPL version 3 or any later version at your choice available at
http://www.gnu.org/licenses/gpl-3.0.txt

*/

#include "includes.h"

#include <arpa/inet.h>
#include <net/if.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/un.h>
#include <unistd.h>

static char* context_version="$Id$";

char *token_autowhite;
char *token_black;
char *token_cctld;
char *token_content;
char *token_context;
char *token_dccbulk;
char *token_dccfrom;
char *token_dccgrey;
char *token_dccto;
char *token_default;
char *token_dnsbl;
char *token_dnsbll;
char *token_envfrom;
char *token_envto;
char *token_filter;
char *token_generic;
char *token_host_limit;
char *token_html_limit;
char *token_html_tags;
char *token_ignore;
char *token_include;
char *token_inherit;
char *token_lbrace;
char *token_mailhost;
char *token_many;
char *token_no;
char *token_off;
char *token_ok;
char *token_ok2;
char *token_on;
char *token_rate;
char *token_rbrace;
char *token_require;
char *token_semi;
char *token_soft;
char *token_spamassassin;
char *token_substitute;
char *token_tld;
char *token_unknown;
char *token_uribl;
char *token_verify;
char *token_white;
char *token_yes;

char *token_myhostname;
#ifndef HOST_NAME_MAX
	#define HOST_NAME_MAX 255
#endif
char myhostname[HOST_NAME_MAX+1];

pthread_mutex_t verifier_mutex; 	// protect the verifier map
verify_map	verifiers;

pthread_mutex_t whitelister_mutex;	// protect the whitelisters map
whitelister_map whitelisters;

string_set	all_strings;	// owns all the strings, only modified by the config loader thread
const int maxlen = 1000;	// used for snprintf buffers
const int maxsmtp_age =  60;// smtp verify sockets older than this are ancient
const int maxauto_age = 600;// auto whitelister delay before flushing to file
extern int	  NULL_SOCKET;
const time_t  ERROR_SMTP_SOCKET_TIME = 600; // number of seconds between attempts to open a socket to an smtp server


int SMTP::writer() {
	#ifdef VERIFY_DEBUG
		log("writer() sees buffer with %s", buffer);
		log("writer() sees error %d", (int)error);
	#endif
	int rs = 0;
	if (!error) {
		int len = strlen(buffer);
		while (rs < len) {
			int ws = write(fd, buffer+rs, len-rs);
			if (ws > 0) {
				rs += ws;
			}
			else {
				// peer closed the socket!
				rs = 0;
				error = true;
				break;
			}
		}
	}
	return rs;
}


int SMTP::reader() {
	// read some bytes terminated by lf or end of buffer.
	// we may have a multi line response or part thereof in the buffer.
	#ifdef VERIFY_DEBUG
		log("reader() sees error %d", (int)error);
	#endif
	if (error) return 0;
	int len = maxlen-1; // room for null terminator
	while (pending < len) {
		int ws = read(fd, buffer+pending, len-pending);
		if (ws > 0) {
			pending += ws;
			if (buffer[pending-1] == '\n') break;
		}
		else {
			// peer closed the socket!
			pending = 0;
			error = true;
			break;
		}
	}
	buffer[pending] = '\0';
	#ifdef VERIFY_DEBUG
		log("reader() sees buffer with %s", buffer);
	#endif
	return pending;
}


int SMTP::read_line() {
	char *lf = strchr(buffer, '\n');
	if (!lf) {
		reader();  // get a lf
		lf = strchr(buffer, '\n');
		if (!lf) lf = buffer + pending - 1;
	}
	return (lf-buffer)+1;	// number of bytes in this line
}


void SMTP::flush_line(int r) {
	if (pending > r) memmove(buffer, buffer+r, pending-r);
	pending -= r;
}


int SMTP::read_response() {
	pending = 0;
	buffer[pending] = '\0';
	while (true) {
		int r = read_line();
		#ifdef VERIFY_DEBUG
			log("read_response() sees line with %s", buffer);
			log("read_response() sees line length %d", r);
		#endif
		if (r == 0) return 0;	// failed to read any bytes
		if ((r > 4) && (buffer[3] == '-')) {
			flush_line(r);
			continue;
		}
		return atoi(buffer);
	}
	return 0;
}


int SMTP::cmd(char *c) {
	if (c) {
		init();
		append(c);
	}
	append("\r\n");
	writer();
	return read_response();
}


int SMTP::helo() {
	if (read_response() != 220) return 0;
	init();
	append("HELO ");
	append(token_myhostname);
	return cmd(NULL);
}


int SMTP::rset() {
	int rc = cmd("RSET");
	efrom[0] = '\0';
	return rc;
}


int SMTP::from(char *f) {
	// the mail from address was originally passed in from sendmail enclosed in
	// <>. to_lower_string() removed the <> and converted the rest to lowercase,
	// except in the case of an empty return path, which was left as the two
	// character string <>.
	if (strncmp(efrom, f, maxlen)) {
		rset();
		strncpy(efrom, f, maxlen);
		init();
		append("MAIL FROM:<");
		if (*f != '<') append(f);
		append(">");
		return cmd(NULL);
	}
	return 250; // pretend it worked
}


int SMTP::rcpt(char *t) {
	init();
	append("RCPT TO:<");
	append(t);
	append(">");
	return cmd(NULL);
}


int SMTP::quit() {
	return cmd("QUIT");
}


void SMTP::closefd() {
	shutdown(fd, SHUT_RDWR);
	close(fd);
}


#ifdef VERIFY_DEBUG
	void SMTP::log(char *m, int v) {
		char buf[maxlen];
		snprintf(buf, maxlen, m, v);
		my_syslog(buf);
	}


	void SMTP::log(char *m, char *v) {
		char buf[maxlen];
		snprintf(buf, maxlen, m, v);
		my_syslog(buf);
	}
#endif


////////////////////////////////////////////////
// smtp verifier so backup mx machines can see the valid users
//
VERIFY::VERIFY(char *h) {
	host	 = h;
	last_err = 0;
	pthread_mutex_init(&mutex, 0);
}


void VERIFY::closer() {
	bool ok = true;
	while (ok) {
		SMTP *conn = NULL;
		pthread_mutex_lock(&mutex);
			if (connections.empty()) {
				ok = false;
			}
			else {
				conn = connections.front();
				time_t now = time(NULL);
				if ((now - conn->get_stamp()) > maxsmtp_age) {
					// this connection is ancient, remove it
					connections.pop_front();
				}
				else {
					ok	 = false;
					conn = NULL;
				}
			}
		pthread_mutex_unlock(&mutex);
		// avoid doing this work inside the mutex lock
		if (conn) {
			#ifdef VERIFY_DEBUG
				conn->log("closer() closes ancient %d", conn->get_fd());
			#endif
			delete conn;
		}
	}
}


SMTP* VERIFY::get_connection() {
	SMTP *conn = NULL;
	pthread_mutex_lock(&mutex);
		if (!connections.empty()) {
			conn = connections.front();
			connections.pop_front();
			#ifdef VERIFY_DEBUG
				conn->log("get_connection() %d from cache", conn->get_fd());
			#endif
		}
	pthread_mutex_unlock(&mutex);
	if (conn) return conn;
	int sock = NULL_SOCKET;
	if ((time(NULL) - last_err) > ERROR_SMTP_SOCKET_TIME) {
		// nothing recent, maybe this time it will work
		hostent *h = gethostbyname(host);
		if (h) {
			sockaddr_in server;
			server.sin_family = h->h_addrtype;
			server.sin_port   = htons(25);
			memcpy(&server.sin_addr, h->h_addr_list[0], h->h_length);
			sock = socket(PF_INET, SOCK_STREAM, 0);
			if (sock != NULL_SOCKET) {
				bool rc = (connect(sock, (sockaddr *)&server, sizeof(server)) == 0);
				if (!rc) {
					shutdown(sock, SHUT_RDWR);
					close(sock);
					sock = NULL_SOCKET;
					last_err = time(NULL);
				}
			}
			else last_err = time(NULL);
		}
		else last_err = time(NULL);
	}
	if (sock != NULL_SOCKET) {
		conn = new SMTP(sock);
		#ifdef VERIFY_DEBUG
			conn->log("get_connection() %d new socket", conn->get_fd());
		#endif
		if (conn->helo() == 250) return conn;
		delete conn;
	}
	return NULL;
}


void VERIFY::put_connection(SMTP *conn) {
	if (conn->err()) {
		#ifdef VERIFY_DEBUG
			conn->log("put_socket() %d with error, close it", conn->get_fd());
		#endif
		delete conn;
		last_err = time(NULL);
	}
	else {
		#ifdef VERIFY_DEBUG
			conn->log("put_socket() %d", conn->get_fd());
		#endif
		conn->now();
		pthread_mutex_lock(&mutex);
			connections.push_back(conn);
		pthread_mutex_unlock(&mutex);
	}
}


bool VERIFY::ok(char *from, char *to) {
	if (host == token_myhostname) return true;
	SMTP *conn = get_connection();
	if (!conn) return true;    // cannot verify right now, we have socket errors
	int rc;
	rc = conn->from(from);
	#ifdef VERIFY_DEBUG
		conn->log("verify::ok() from sees %d", rc);
	#endif
	if (rc != 250) {
		conn->rset();
		put_connection(conn);
		return (rc >= 500) ? false : true;
	}
	rc = conn->rcpt(to);
	#ifdef VERIFY_DEBUG
		conn->log("verify::ok() rcpt sees %d", rc);
	#endif
	put_connection(conn);
	return (rc >= 500) ? false : true;
}


////////////////////////////////////////////////
// setup a new smtp verify host
//
VERIFYP add_verify_host(char *host);
VERIFYP add_verify_host(char *host) {
	VERIFYP rc = NULL;
	pthread_mutex_lock(&verifier_mutex);
		verify_map::iterator i = verifiers.find(host);
		if (i == verifiers.end()) {
			rc = new VERIFY(host);
			verifiers[host] = rc;
		}
		else rc = (*i).second;
	pthread_mutex_unlock(&verifier_mutex);
	return rc;
}


////////////////////////////////////////////////
// thread to check for verify hosts with old sockets that we can close
//
void* verify_closer(void *arg) {
	while (true) {
		sleep(maxsmtp_age);
		pthread_mutex_lock(&verifier_mutex);
			for (verify_map::iterator i=verifiers.begin(); i!=verifiers.end(); i++) {
				VERIFYP v = (*i).second;
				v->closer();
			}
		pthread_mutex_unlock(&verifier_mutex);
	}
	return NULL;
}


////////////////////////////////////////////////
// automatic whitelister
//
WHITELISTER::WHITELISTER(char *f, int d) {
	fn		 = f;
	days	 = d;
	pthread_mutex_init(&mutex, 0);
	need	 = false;
	loaded	 = time(NULL);
	merge();
}


void WHITELISTER::merge() {
	time_t now = time(NULL);
	ifstream ifs;
	ifs.open(fn);
	if (!ifs.fail()) {
		const int maxlen = 1000;
		char buf[maxlen];
		while (ifs.getline(buf, maxlen)) {
			char *p = strchr(buf, ' ');
			if (p) {
				*p = '\0';
				char   *who = strdup(buf);
				time_t when = atoi(p+1);
				if ((when == 0) || (when > now)) when = now;
				autowhite_sent::iterator i = rcpts.find(who);
				if (i == rcpts.end()) {
					rcpts[who] = when;
				}
				else {
					time_t wh = (*i).second;
					if ((when == 1) || (when > wh)) (*i).second = when;
					free(who);
				}
			}
		}
	}
	ifs.close();
}


void WHITELISTER::writer() {
	pthread_mutex_lock(&mutex);
		time_t limit = time(NULL) - days*86400;

		// check for manually modified autowhitelist file
		struct stat st;
		if (stat(fn, &st)) need = true; // file has disappeared
		else if (st.st_mtime > loaded) {
			// file has been manually updated, merge new entries
			merge();
			need = true;
		}

		// purge old entries
		for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end();) {
			time_t when = (*i).second;
			if (when < limit) {
				char *who = (*i).first;
				free(who);
				autowhite_sent::iterator j = i;
				j++;
				rcpts.erase(i);
				i = j;
				need = true;
			}
			else i++;
		}

		if (need) {
			// dump the file
			ofstream ofs;
			ofs.open(fn);
			if (!ofs.fail()) {
				for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end(); i++) {
					char *who = (*i).first;
					int  when = (*i).second;
					if (!strchr(who, ' ')) {
						ofs << who << " " << when << endl;
					}
				}
			}
			ofs.close();
			need = false;
			loaded = time(NULL);	// update load time
		}
	pthread_mutex_unlock(&mutex);
}


void WHITELISTER::sent(char *to) {
	// we take ownership of the string
	pthread_mutex_lock(&mutex);
		need = true;
		autowhite_sent::iterator i = rcpts.find(to);
		if (i == rcpts.end()) {
			rcpts[to] = time(NULL);
		}
		else {
			(*i).second = time(NULL);
			free(to);
		}
	pthread_mutex_unlock(&mutex);
}


bool WHITELISTER::is_white(char *from) {
	pthread_mutex_lock(&mutex);
		autowhite_sent::iterator i = rcpts.find(from);
		bool rc = (i != rcpts.end());
	pthread_mutex_unlock(&mutex);
	return rc;
}


////////////////////////////////////////////////
// setup a new auto whitelister file
//
WHITELISTERP add_whitelister_file(char *fn, int days);
WHITELISTERP add_whitelister_file(char *fn, int days) {
	WHITELISTERP rc = NULL;
	pthread_mutex_lock(&whitelister_mutex);
		whitelister_map::iterator i = whitelisters.find(fn);
		if (i == whitelisters.end()) {
			rc = new WHITELISTER(fn, days);
			whitelisters[fn] = rc;
		}
		else {
			rc = (*i).second;
			rc->set_days(days);
		}
	pthread_mutex_unlock(&whitelister_mutex);
	return rc;
}


////////////////////////////////////////////////
// thread to check for whitelister hosts with old sockets that we can close
//
void* whitelister_writer(void *arg) {
	while (true) {
		sleep(maxauto_age);
		pthread_mutex_lock(&whitelister_mutex);
			for (whitelister_map::iterator i=whitelisters.begin(); i!=whitelisters.end(); i++) {
				WHITELISTERP v = (*i).second;
				v->writer();
			}
		pthread_mutex_unlock(&whitelister_mutex);
	}
	return NULL;
}


DNSBL::DNSBL(char *n, char *s, char *m) {
	name	= n;
	suffix	= s;
	message = m;
}


bool DNSBL::operator==(const DNSBL &rhs) {
	return (strcmp(name,	rhs.name)	 == 0) &&
		   (strcmp(suffix,	rhs.suffix)  == 0) &&
		   (strcmp(message, rhs.message) == 0);
}


CONFIG::CONFIG() {
	reference_count    = 0;
	generation		   = 0;
	load_time		   = 0;
	default_context    = NULL;
}


CONFIG::~CONFIG() {
	if (debug_syslog) {
		char buf[maxlen];
		snprintf(buf, sizeof(buf), "freeing memory for old configuration generation %d", generation);
		my_syslog(buf);
	}
	for (context_list::iterator i=contexts.begin(); i!=contexts.end(); i++) {
		CONTEXT *c = *i;
		delete c;
	}
}


void CONFIG::add_context(CONTEXTP con) {
	contexts.push_back(con);
	if (!default_context && !con->get_parent()) {
		// first global context
		default_context = con;
	}
}


void CONFIG::add_to(char *to, CONTEXTP con) {
	context_map::iterator i = env_to.find(to);
	if (i != env_to.end()) {
		CONTEXTP  c = (*i).second;
		if ((c != con) && (c != con->get_parent())) {
			if (debug_syslog) {
				char oldname[maxlen];
				char newname[maxlen];
				char *oldn = c->get_full_name(oldname, maxlen);
				char *newn = con->get_full_name(newname, maxlen);
				char buf[maxlen*3];
				snprintf(buf, maxlen*3, "both %s and %s claim envelope to %s, the second one wins", oldn, newn, to);
				my_syslog(buf);
			}
		}
	}
	env_to[to] = con;
}


CONTEXTP CONFIG::find_context(char *to) {
	context_map::iterator i = env_to.find(to);
	if (i != env_to.end()) return (*i).second;		// found user@domain key
	char *x = strchr(to, '@');
	if (x) {
		x++;
		i = env_to.find(x);
		if (i != env_to.end()) return (*i).second;	// found domain key
		char y = *x;
		*x = '\0';
		i = env_to.find(to);
		*x = y;
		if (i != env_to.end()) return (*i).second;	// found user@ key
	}
	return default_context;
}


void CONFIG::dump() {
	bool spamass = false;
	if (default_context) default_context->dump(true, spamass);
	for (context_list::iterator i=contexts.begin(); i!=contexts.end(); i++) {
		CONTEXTP c = *i;
		CONTEXTP p = c->get_parent();
		if (!p && (c != default_context)) c->dump(false, spamass);
	}
	char buf[maxlen];
	for (context_map::iterator i=env_to.begin(); i!=env_to.end(); i++) {
		char	 *to = (*i).first;
		CONTEXTP con = (*i).second;
		printf("// envelope to %s \t-> context %s \n", to, con->get_full_name(buf,maxlen));
	}
	if (spamass && (spamc == spamc_empty)) {
		printf("// *** warning - spamassassin filtering requested, but spamc not found by autoconf.\n");
	}
}


CONTEXT::CONTEXT(CONTEXTP parent_, char *name_) {
	parent				= parent_;
	name				= name_;
	verify_host 		= NULL;
	verifier			= NULL;
	generic_regx		= NULL;
	generic_message 	= NULL;
	autowhite_file		= NULL;
	whitelister 		= NULL;
	env_from_default	= (parent) ? token_inherit : token_unknown;
	content_filtering	= (parent) ? parent->content_filtering : false;
	content_suffix		= NULL;
	content_message 	= NULL;
	uribl_suffix		= NULL;
	uribl_message		= NULL;
	host_limit			= (parent) ? parent->host_limit  : 0;
	host_limit_message	= NULL;
	host_random 		= (parent) ? parent->host_random : false;
	tag_limit			= (parent) ? parent->tag_limit	 : 0;
	tag_limit_message	= NULL;
	spamassassin_limit	= (parent) ? parent->spamassassin_limit : 0;
	require_match		= (parent) ? parent->require_match		: false;
	dcc_greylist		= (parent) ? parent->dcc_greylist		: false;
	dcc_bulk_threshold	= (parent) ? parent->dcc_bulk_threshold : 0;
	default_rcpt_rate	= INT_MAX;
}


CONTEXT::~CONTEXT() {
	for (dnsblp_map::iterator i=dnsbl_names.begin(); i!=dnsbl_names.end(); i++) {
		DNSBLP d = (*i).second;
		// delete the underlying DNSBL objects.
		delete d;
	}
	if (generic_regx) regfree(&generic_pattern);
}


bool CONTEXT::is_parent(CONTEXTP p) {
	if (p == parent) return true;
	if (!parent) return false;
	return parent->is_parent(p);
}


char *CONTEXT::get_full_name(char *buffer, int size) {
	if (!parent) return name;
	char buf[maxlen];
	snprintf(buffer, size, "%s.%s", parent->get_full_name(buf, maxlen), name);
	return buffer;
}


bool CONTEXT::set_generic(char *regx, char *msg)
{
	int rc = 0;
	if (generic_regx) regfree(&generic_pattern);
	generic_regx	= regx;
	generic_message = msg;
	if (generic_regx) {
		rc = regcomp(&generic_pattern, regx, REG_NOSUB | REG_ICASE | REG_EXTENDED);
	}
	return rc;	// true iff bad pattern
}


char *CONTEXT::generic_match(char *client)
{
	if (parent && !generic_regx) return parent->generic_match(client);
	if (!generic_regx)			 return NULL;
	if (0 == regexec(&generic_pattern, client, 0, NULL, 0)) {
		return generic_message;
	}
	return NULL;
}


bool CONTEXT::cover_env_to(char *to) {
	char buffer[maxlen];
	char *x = strchr(to, '@');
	if (x) x++;
	else   x = to;
	if (*x == '\0') return true;                // always allow covering addresses with no domain name, eg abuse@
	if (!parent && env_to.empty()) return true; // empty env_to at global level covers everything
	string_set::iterator i = env_to.find(x);
	if (i != env_to.end()) return true; 		// we cover the entire domain
	if (x != to) {
		i = env_to.find(to);
		if (i != env_to.end()) return true; 	// we cover the specific email address
	}
	return false;
}


VERIFYP CONTEXT::find_verify(char *to) {
	if (verifier && (verify_host != token_myhostname) && cover_env_to(to))
		return verifier;
	else if (parent)
		return parent->find_verify(to);
	else
		return NULL;
}


WHITELISTERP CONTEXT::find_autowhite(char *from, char *to) {
	if (whitelister && cover_env_to(to) && !cover_env_to(from))
		return whitelister;
	else if (parent)
		return parent->find_autowhite(from, to);
	else
		return NULL;
}


int CONTEXT::find_rate(char *user) {
	if (rcpt_per_hour.empty()) return default_rcpt_rate;
	rcpt_rates::iterator i = rcpt_per_hour.find(user);
	return (i == rcpt_per_hour.end()) ? default_rcpt_rate : (*i).second;
}


char *CONTEXT::find_from(char *from, bool update_white) {
	if (whitelister && whitelister->is_white(from)) {
		if (update_white) {
			// update senders timestamp to extend the whitelisting period
			if (debug_syslog > 1) {
				char buf[maxlen];
				char msg[maxlen];
				snprintf(msg, sizeof(msg), "extend whitelist reply from <%s> in context %s", from, get_full_name(buf,maxlen));
				my_syslog(msg);
			}
			whitelister->sent(strdup(from));
		}
		return token_white;
	}
	char *rc = env_from_default;
	string_map::iterator i = env_from.find(from);
	if (i != env_from.end()) rc = (*i).second;	// found user@domain key
	else {
		char *x = strchr(from, '@');
		if (x) {
			x++;
			i = env_from.find(x);
			if (i != env_from.end()) rc = (*i).second;	// found domain key
			else {
				char y = *x;
				*x = '\0';
				i = env_from.find(from);
				*x = y;
				if (i != env_from.end()) rc = (*i).second;	// found user@ key
			}
		}
	}
	if ((rc == token_inherit) && parent) return parent->find_from(from);
	return (rc == token_inherit) ? token_unknown : rc;
}


CONTEXTP CONTEXT::find_context(char *from) {
	context_map::iterator i = env_from_context.find(from);
	if (i != env_from_context.end()) return (*i).second;		// found user@domain key
	char *x = strchr(from, '@');
	if (x) {
		x++;
		i = env_from_context.find(x);
		if (i != env_from_context.end()) return (*i).second;	// found domain key
		char y = *x;
		*x = '\0';
		i = env_from_context.find(from);
		*x = y;
		if (i != env_from_context.end()) return (*i).second;			  // found user@ key
	}
	return this;
}


CONTEXTP CONTEXT::find_from_context_name(char *name) {
	context_map::iterator i = children.find(name);
	if (i != children.end()) return (*i).second;
	return NULL;
}


DNSBLP CONTEXT::find_dnsbl(char *name) {
	dnsblp_map::iterator i = dnsbl_names.find(name);
	if (i != dnsbl_names.end()) return (*i).second;
	if (parent) return parent->find_dnsbl(name);
	return NULL;
}


char* CONTEXT::get_content_suffix() {
	if (!content_suffix && parent) return parent->get_content_suffix();
	return content_suffix;
}


char* CONTEXT::get_uribl_suffix() {
	if (!uribl_suffix && parent) return parent->get_uribl_suffix();
	return uribl_suffix;
}


char* CONTEXT::get_content_message() {
	if (!content_message && parent) return parent->get_content_message();
	return content_message;
}


char* CONTEXT::get_uribl_message() {
	if (!uribl_message && parent) return parent->get_uribl_message();
	return uribl_message;
}


string_set& CONTEXT::get_content_host_ignore() {
	if (content_host_ignore.empty() && parent) return parent->get_content_host_ignore();
	return content_host_ignore;
}


string_set& CONTEXT::get_content_cctlds() {
	if (content_cctlds.empty() && parent) return parent->get_content_cctlds();
	return content_cctlds;
}

string_set& CONTEXT::get_content_tlds() {
	if (content_tlds.empty() && parent) return parent->get_content_tlds();
	return content_tlds;
}


string_set& CONTEXT::get_html_tags() {
	if (html_tags.empty() && parent) return parent->get_html_tags();
	return html_tags;
}


dnsblp_list& CONTEXT::get_dnsbl_list() {
	if (dnsbl_list.empty() && parent) return parent->get_dnsbl_list();
	return dnsbl_list;
}


bool CONTEXT::acceptable_content(recorder &memory, int score, int bulk, string& msg) {
	if (spamassassin_limit && (score > spamassassin_limit)) {
		char buf[maxlen];
		snprintf(buf, sizeof(buf), "Mail rejected - spam assassin score %d", score);
		msg = string(buf);
		return false;
	}
	if (dcc_bulk_threshold && (bulk > dcc_bulk_threshold)) {
		char buf[maxlen];
		snprintf(buf, sizeof(buf), "Mail rejected - dcc score %d", bulk);
		msg = string(buf);
		return false;
	}
	if (memory.excessive_bad_tags(tag_limit)) {
		msg = string(tag_limit_message);
		return false;
	}
	if (!host_random && memory.excessive_hosts(host_limit)) {
		msg = string(host_limit_message);
		return false;
	}
	return true;
}


void CONTEXT::dump(bool isdefault, bool &spamass, int level) {
	char indent[maxlen];
	int i = min(maxlen-1, level*4);
	memset(indent, ' ', i);
	indent[i] = '\0';
	char buf[maxlen];
	char *fullname = get_full_name(buf,maxlen);
	printf("%s context %s { \t// %s\n", indent, name, fullname);

	for (dnsblp_map::iterator i=dnsbl_names.begin(); i!=dnsbl_names.end(); i++) {
		char *n = (*i).first;
		DNSBL &d = *(*i).second;
		printf("%s     dnsbl %s %s \"%s\"; \n", indent, n, d.suffix, d.message);
	}

	dnsblp_list dl = get_dnsbl_list();
	if (!dl.empty()) {
		printf("%s     dnsbl_list", indent);
		for (dnsblp_list::iterator i=dl.begin(); i!=dl.end(); i++) {
			DNSBL &d = *(*i);
			printf(" %s", d.name);
		}
		printf("; \n");
	}

	if (content_filtering) {
		printf("%s     content on { \n", indent, env_from_default);
		if (content_suffix) {
			printf("%s         filter %s \"%s\"; \n", indent, content_suffix, content_message);
		}
		if (uribl_suffix) {
			printf("%s         uribl %s \"%s\"; \n", indent, uribl_suffix, uribl_message);
		}
		if (!content_host_ignore.empty()) {
			printf("%s         ignore { \n", indent);
			for (string_set::iterator i=content_host_ignore.begin(); i!=content_host_ignore.end(); i++) {
				printf("%s             %s; \n", indent, *i);
			}
			printf("%s         }; \n", indent);
		}
		if (!content_cctlds.empty()) {
			printf("%s         cctld { \n", indent);
			printf("%s             ", indent);
			for (string_set::iterator i=content_cctlds.begin(); i!=content_cctlds.end(); i++) {
				printf("%s; ", *i);
			}
			printf("\n%s         }; \n", indent);
		}
		if (!content_tlds.empty()) {
			printf("%s         tld { \n", indent);
			printf("%s             ", indent);
			for (string_set::iterator i=content_tlds.begin(); i!=content_tlds.end(); i++) {
				printf("%s; ", *i);
			}
			printf("\n%s         }; \n", indent);
		}
		if (!html_tags.empty()) {
			printf("%s         html_tags { \n", indent);
			printf("%s             ", indent);
			for (string_set::iterator i=html_tags.begin(); i!=html_tags.end(); i++) {
				printf("%s; ", *i);
			}
			printf("\n%s         }; \n", indent);
		}
		if (host_limit_message) {
			printf("%s         host_limit on %d \"%s\"; \n", indent, host_limit, host_limit_message);
		}
		else if (host_random) {
			printf("%s         host_limit soft %d; \n", indent, host_limit);
		}
		else {
			printf("%s         host_limit off; \n", indent);
		}
		if (tag_limit_message) {
			printf("%s         html_limit on %d \"%s\"; \n", indent, tag_limit, tag_limit_message);
		}
		else {
			printf("%s         html_limit off; \n", indent);
		}
		printf("%s         spamassassin %d; \n", indent, spamassassin_limit);
		printf("%s         require_match %s; \n", indent, (require_match) ? "yes" : "no");
		printf("%s         dcc_greylist  %s; \n", indent, (dcc_greylist)  ? "yes" : "no");
		if (dcc_bulk_threshold == 0)			printf("%s         dcc_bulk_threshold off; \n", indent);
		else if (dcc_bulk_threshold >= dccbulk) printf("%s         dcc_bulk_threshold many; \n", indent);
		else									printf("%s         dcc_bulk_threshold %d; \n", indent, dcc_bulk_threshold);
		printf("%s     }; \n", indent);
		spamass |= (spamassassin_limit != 0);
		}
	else {
		printf("%s     content off {}; \n", indent, env_from_default);
	}

	printf("%s     env_to { \t// %s\n", indent, fullname);
	for (string_set::iterator i=env_to.begin(); i!=env_to.end(); i++) {
		printf("%s         %s; \n", indent, *i);
	}
	printf("%s     }; \n", indent);

	if (verify_host) {
		printf("%s     verify %s; \n", indent, verify_host);
	}

	if (generic_regx) {
		printf("%s     generic \"%s\"  \n", indent, generic_regx);
		printf("%s             \"%s\"; \n", indent, generic_message);
	}

	if (autowhite_file && whitelister) {
		printf("%s     autowhite %d %s; \n", indent, whitelister->get_days(), autowhite_file);
	}

	for (context_map::iterator i=children.begin(); i!=children.end(); i++) {
		CONTEXTP c = (*i).second;
		c->dump(false, spamass, level+1);
	}

	printf("%s     env_from %s { \t// %s\n", indent, env_from_default, fullname);
	if (!env_from.empty()) {
		printf("%s     // white/black/unknown \n", indent);
		for (string_map::iterator i=env_from.begin(); i!=env_from.end(); i++) {
			char *f = (*i).first;
			char *t = (*i).second;
			printf("%s         %s \t%s; \n", indent, f, t);
		}
	}
	if (!env_from_context.empty()) {
		printf("%s     // child contexts \n", indent);
		for (context_map::iterator j=env_from_context.begin(); j!=env_from_context.end(); j++) {
			char	*f = (*j).first;
			CONTEXTP t = (*j).second;
			printf("%s         %s \t%s; \n", indent, f, t->name);
		}
	}
	printf("%s     }; \n", indent);

	if (isdefault) {
		printf("%s     rate_limit %d { \n", indent, default_rcpt_rate);
		for (rcpt_rates::iterator j=rcpt_per_hour.begin(); j!=rcpt_per_hour.end(); j++) {
			char	*u = (*j).first;
			int 	 l = (*j).second;
			printf("%s         \"%s\" \t%d; \n", indent, u, l);
		}
		printf("%s     }; \n", indent);
	}

	printf("%s }; \n", indent);
}


////////////////////////////////////////////////
// helper to discard the strings held by a string_set
//
void discard(string_set &s) {
	for (string_set::iterator i=s.begin(); i!=s.end(); i++) {
		free(*i);
	}
	s.clear();
}


////////////////////////////////////////////////
// helper to register a string in a string set
//
char* register_string(string_set &s, char *name) {
	string_set::iterator i = s.find(name);
	if (i != s.end()) return *i;
	char *x = strdup(name);
	s.insert(x);
	return x;
}


////////////////////////////////////////////////
// register a global string
//
char* register_string(char *name) {
	return register_string(all_strings, name);
}


////////////////////////////////////////////////
// clear all global strings, helper for valgrind checking
//
void clear_strings() {
	discard(all_strings);
}


////////////////////////////////////////////////
//
bool tsa(TOKEN &tok, char *token);
bool tsa(TOKEN &tok, char *token) {
	char *have = tok.next();
	if (have == token) return true;
	tok.token_error(token, have);
	return false;
}


////////////////////////////////////////////////
//
bool parse_dnsbl(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnsbl(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	char *name = tok.next();
	char *suf = tok.next();
	char *msg = tok.next();
	if (!tsa(tok, token_semi)) return false;
	DNSBLP dnsnew = new DNSBL(name, suf, msg);
	DNSBLP dnsold = me.find_dnsbl(name);
	if (dnsold && (*dnsold == *dnsnew)) {
		// duplicate redefinition, ignore it
		delete dnsnew;
		return true;
	}
	me.add_dnsbl(name, dnsnew);
	return true;
}


////////////////////////////////////////////////
//
bool parse_dnsbll(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnsbll(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	while (true) {
		char *have = tok.next();
		if (!have) break;
		if (have == token_semi) break;
		DNSBLP dns = me.find_dnsbl(have);
		if (dns) {
			me.add_dnsbl(dns);
		}
		else {
			tok.token_error("dnsbl name", have);
			return false;
		}
	}
	return true;
}


////////////////////////////////////////////////
//
bool parse_content(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_content(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	char *setting = tok.next();
	if (setting == token_on) {
		me.set_content_filtering(true);
	}
	else if (setting == token_off) {
		me.set_content_filtering(false);
	}
	else {
		tok.token_error("on/off", setting);
		return false;
	}
	if (!tsa(tok, token_lbrace)) return false;
	while (true) {
		char *have = tok.next();
		if (!have) break;
		if (have == token_filter) {
			char *suffix = tok.next();
			char *messag = tok.next();
			me.set_content_suffix(suffix);
			me.set_content_message(messag);
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_uribl) {
			char *suffix = tok.next();
			char *messag = tok.next();
			me.set_uribl_suffix(suffix);
			me.set_uribl_message(messag);
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_ignore) {
			if (!tsa(tok, token_lbrace)) return false;
			while (true) {
				if (!have) break;
				char *have = tok.next();
				if (have == token_rbrace) break;  // done
				me.add_ignore(have);
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_tld) {
			if (!tsa(tok, token_lbrace)) return false;
			while (true) {
				char *have = tok.next();
				if (!have) break;
				if (have == token_rbrace) break;  // done
				me.add_tld(have);
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_cctld) {
			if (!tsa(tok, token_lbrace)) return false;
			while (true) {
				char *have = tok.next();
				if (!have) break;
				if (have == token_rbrace) break;  // done
				me.add_cctld(have);
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_html_tags) {
			if (!tsa(tok, token_lbrace)) return false;
			while (true) {
				char *have = tok.next();
				if (!have) break;
				if (have == token_rbrace) {
					break;	// done
				}
				else {
					me.add_tag(have);							// base version
					char buf[200];
					snprintf(buf, sizeof(buf), "/%s", have);
					me.add_tag(register_string(buf));			// leading /
					snprintf(buf, sizeof(buf), "%s/", have);
					me.add_tag(register_string(buf));			// trailing /
				}
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_html_limit) {
			have = tok.next();
			if (have == token_on) {
				me.set_tag_limit(tok.nextint());
				me.set_tag_message(tok.next());
			}
			else if (have == token_off) {
				me.set_tag_limit(0);
				me.set_tag_message(NULL);
			}
			else {
				tok.token_error("on/off", have);
				return false;
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_host_limit) {
			have = tok.next();
			if (have == token_on) {
				me.set_host_limit(tok.nextint());
				me.set_host_message(tok.next());
				me.set_host_random(false);
			}
			else if (have == token_off) {
				me.set_host_limit(0);
				me.set_host_message(NULL);
				me.set_host_random(false);
			}
			else if (have == token_soft) {
				me.set_host_limit(tok.nextint());
				me.set_host_message(NULL);
				me.set_host_random(true);
			}
			else {
				tok.token_error("on/off/soft", have);
				return false;
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_spamassassin) {
			me.set_spamassassin_limit(tok.nextint());
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_require) {
			have = tok.next();
				 if (have == token_yes) me.set_require(true);
			else if (have == token_no)	me.set_require(false);
			else {
				tok.token_error("yes/no", have);
				return false;
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_dccgrey) {
			have = tok.next();
				 if (have == token_yes) me.set_grey(true);
			else if (have == token_no)	me.set_grey(false);
			else {
				tok.token_error("yes/no", have);
				return false;
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_dccbulk) {
			have = tok.next();
				 if (have == token_off) me.set_bulk(0);
			else if (have == token_many) me.set_bulk(dccbulk);
			else {
				char *e;
				long i = strtol(have, &e, 10);
				if (*e != '\0') {
					tok.token_error("integer", have);
					return false;
				}
				me.set_bulk((int)i);
			}
			if (!tsa(tok, token_semi)) return false;
		}
		else if (have == token_rbrace) {
			break;	// done
		}
		else {
			tok.token_error("content keyword", have);
			return false;
		}
	}
	return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_envto(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_envto(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	if (!tsa(tok, token_lbrace)) return false;
	while (true) {
		char *have = tok.next();
		if (!have) break;
		if (have == token_rbrace) break;
		if (have == token_semi) {
			// optional separators
		}
		else if (have == token_dccto) {
			char *flavor = tok.next();
			if (!tsa(tok, token_lbrace)) return false;
			bool keeping = false;
			while (true) {
				char *have = tok.next();
				if (!have) break;
				if (have == token_rbrace) break;
				if (have == flavor) {
					keeping = true;
					continue;
				}
				else if ((have == token_ok) || (have == token_ok2) || (have == token_many)) {
					keeping = false;
					continue;
				}
				if (have == token_envto) {
					have = tok.next();
					if (keeping) {
						if (me.allow_env_to(have)) {
							me.add_to(have);
							dc.add_to(have, &me);
						}
					}
				}
			  //else if (have == token_substitute) {
			  //	if (tok.next() == token_mailhost) {
			  //		have = tok.next();
			  //		if (keeping) {
			  //			if (me.allow_env_to(have)) {
			  //				me.add_to(have);
			  //				dc.add_to(have, &me);
			  //			}
			  //		}
			  //	}
			  //}
				tok.skipeol();
			}
		}
		else if (me.allow_env_to(have)) {
			me.add_to(have);
			dc.add_to(have, &me);
		}
		else {
			tok.token_error("user@ or user@domain.tld or domain.tld where domain.tld allowed by parent context", have);
			return false;
		}
	}
	return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_verify(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_verify(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	char *host = tok.next();
	if (!tsa(tok, token_semi)) return false;
	me.set_verify(host);
	me.set_verifier(add_verify_host(host));
	return true;
}


////////////////////////////////////////////////
//
bool parse_generic(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_generic(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	char *regx = tok.next();
	char *msg  = tok.next();
	if (!tsa(tok, token_semi)) return false;
	if (me.set_generic(regx, msg)) {
		tok.token_error("invalid regular expression %s", regx, regx);
		return false;
	}
	return true;
}


////////////////////////////////////////////////
//
bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	int days = tok.nextint();
	char *fn = tok.next();
	if (!tsa(tok, token_semi)) return false;
	me.set_autowhite(fn);
	me.set_whitelister(add_whitelister_file(fn, days));
	return true;
}


////////////////////////////////////////////////
//
bool parse_envfrom(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_envfrom(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	char *st = tok.next();
	if ((st == token_black) || (st == token_white) || (st == token_unknown) || (st == token_inherit)) {
		me.set_from_default(st);
	}
	else {
		tok.push(st);
	}
	if (!tsa(tok, token_lbrace)) return false;
	while (true) {
		char *have = tok.next();
		if (!have) break;
		if (have == token_rbrace) break;
		if (have == token_semi) {
			// optional separators
		}
		else if (have == token_dccfrom) {
			if (!tsa(tok, token_lbrace)) return false;
			bool keeping = false;
			bool many = false;
			while (true) {
				char *have = tok.next();
				if (!have) break;
				if (have == token_rbrace) break;
				if (have == token_ok) {
					keeping = true;
					many	= false;
					continue;
				}
				else if (have == token_many) {
					keeping = true;
					many	= true;
					continue;
				}
				else if (have == token_ok2) {
					keeping = false;
					continue;
				}
				if (have == token_envfrom) {
					have = tok.next();
					if (keeping) {
						me.add_from(have, (many) ? token_black : token_white);
					}
				}
				else if (have == token_substitute) {
					if (tok.next() == token_mailhost) {
						have = tok.next();
						me.add_from(have, (many) ? token_black : token_white);
					}
				}
				tok.skipeol();
			}
		}
		else {
			// may be a valid email address or domain name
			char *st = tok.next();
			if ((st == token_white) || (st == token_black) || (st == token_unknown) || (st == token_inherit)) {
				me.add_from(have, st);
			}
			else {
				CONTEXTP con = me.find_from_context_name(st);
				if (con) {
					me.add_from_context(have, con);
				}
				else {
					tok.token_error("white/black/unknown/inherit or child context name", st);
					return false;
				}
			}
		}
	}
	return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
	char *def = tok.next();
	tok.push(def);
	if (def != token_lbrace) me.set_default_rate(tok.nextint());
	if (!tsa(tok, token_lbrace)) return false;
	while (true) {
		char *have = tok.next();
		if (!have) break;
		if (have == token_rbrace) break;
		if (have == token_semi) {
			// optional separators
		}
		else {
			me.add_rate(have, tok.nextint());
		}
	}
	return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_context(TOKEN &tok, CONFIG &dc, CONTEXTP parent);
bool parse_context(TOKEN &tok, CONFIG &dc, CONTEXTP parent) {
	char *name = tok.next();
	if (!tsa(tok, token_lbrace)) return false;
	CONTEXTP con = new CONTEXT(parent, name);

	while (true) {
		char *have = tok.next();
		if (!have) break;
		if (have == token_rbrace) break;  // done
		if (have == token_dnsbl) {
			if (!parse_dnsbl(tok, dc, *con)) return false;
		}
		else if (have == token_dnsbll) {
			if (!parse_dnsbll(tok, dc, *con)) return false;
		}
		else if (have == token_content) {
			if (!parse_content(tok, dc, *con)) return false;
		}
		else if (have == token_envto) {
			if (!parse_envto(tok, dc, *con)) return false;
		}
		else if (have == token_verify) {
			if (!parse_verify(tok, dc, *con)) return false;
		}
		else if (have == token_generic) {
			if (!parse_generic(tok, dc, *con)) return false;
		}
		else if (have == token_autowhite) {
			if (!parse_autowhite(tok, dc, *con)) return false;
		}
		else if (have == token_envfrom) {
			if (!parse_envfrom(tok, dc, *con)) return false;
		}
		else if (have == token_rate) {
			if (parent || dc.default_context) tok.token_error("rate limit ignored in non default context");
			if (!parse_rate(tok, dc, *con)) return false;
		}
		else if (have == token_context) {
			if (!parse_context(tok, dc, con)) return false;
		}
		else {
			tok.token_error("context keyword", have);
			return false;
		}
	}

	if (!tsa(tok, token_semi)) {
		delete con;
		return false;
	}
	dc.add_context(con);
	if (parent) parent->add_context(con);
	return true;
}


////////////////////////////////////////////////
// parse a config file
//
bool load_conf(CONFIG &dc, char *fn) {
	int count = 0;
	TOKEN tok(fn, &dc.config_files);
	while (true) {
		char *have = tok.next();
		if (!have) break;
		if (have == token_context) {
			if (!parse_context(tok, dc, NULL)) {
				tok.token_error("load_conf() failed to parse context");
				return false;
			}
			else count++;
		}
		else {
			tok.token_error(token_context, have);
			return false;
		}
	}
	tok.token_error("load_conf() found %d contexts in %s", count, fn);
	return (dc.default_context) ? true : false;
}


////////////////////////////////////////////////
// init the tokens
//
void token_init() {
	token_autowhite 	= register_string("autowhite");
	token_black 		= register_string("black");
	token_cctld 		= register_string("cctld");
	token_content		= register_string("content");
	token_context		= register_string("context");
	token_dccbulk		= register_string("dcc_bulk_threshold");
	token_dccfrom		= register_string("dcc_from");
	token_dccgrey		= register_string("dcc_greylist");
	token_dccto 		= register_string("dcc_to");
	token_default		= register_string("default");
	token_dnsbl 		= register_string("dnsbl");
	token_dnsbll		= register_string("dnsbl_list");
	token_envfrom		= register_string("env_from");
	token_envto 		= register_string("env_to");
	token_filter		= register_string("filter");
	token_generic		= register_string("generic");
	token_host_limit	= register_string("host_limit");
	token_html_limit	= register_string("html_limit");
	token_html_tags 	= register_string("html_tags");
	token_ignore		= register_string("ignore");
	token_include		= register_string("include");
	token_inherit		= register_string("inherit");
	token_lbrace		= register_string("{");
	token_mailhost		= register_string("mail_host");
	token_many			= register_string("many");
	token_no			= register_string("no");
	token_off			= register_string("off");
	token_ok			= register_string("ok");
	token_ok2			= register_string("ok2");
	token_on			= register_string("on");
	token_rate			= register_string("rate_limit");
	token_rbrace		= register_string("}");
	token_require		= register_string("require_match");
	token_semi			= register_string(";");
	token_soft			= register_string("soft");
	token_spamassassin	= register_string("spamassassin");
	token_substitute	= register_string("substitute");
	token_tld			= register_string("tld");
	token_unknown		= register_string("unknown");
	token_uribl 		= register_string("uribl");
	token_verify		= register_string("verify");
	token_white 		= register_string("white");
	token_yes			= register_string("yes");

	if (gethostname(myhostname, HOST_NAME_MAX+1) != 0) {
		strncpy(myhostname, "localhost", HOST_NAME_MAX+1);
	}
	myhostname[HOST_NAME_MAX] = '\0'; // ensure null termination
	token_myhostname = register_string(myhostname);
}