/*

Copyright (c) 2013 Carl Byington - 510 Software Group, released under
the GPL version 3 or any later version at your choice available at
http://www.gnu.org/licenses/gpl-3.0.txt

*/

#include "includes.h"

#include <arpa/inet.h>
#include <arpa/nameser.h>
#include <net/if.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/un.h>
#include <unistd.h>
#include <climits>

const char *token_asterisk;
const char *token_autowhite;
const char *token_bang;
const char *token_black;
const char *token_content;
const char *token_context;
const char *token_dccbulk;
const char *token_dccfrom;
const char *token_dccgrey;
const char *token_dccto;
const char *token_default;
const char *token_dnsbl;
const char *token_dnsbll;
const char *token_dnswl;
const char *token_dnswll;
const char *token_envfrom;
const char *token_envto;
const char *token_filter;
const char *token_generic;
const char *token_host_limit;
const char *token_html_limit;
const char *token_html_tags;
const char *token_ignore;
const char *token_include;
const char *token_inherit;
const char *token_lbrace;
const char *token_mailhost;
const char *token_many;
const char *token_no;
const char *token_off;
const char *token_ok;
const char *token_ok2;
const char *token_on;
const char *token_period;
const char *token_rate;
const char *token_rbrace;
const char *token_require;
const char *token_requirerdns;
const char *token_semi;
const char *token_soft;
const char *token_spamassassin;
const char *token_substitute;
const char *token_tld;
const char *token_unknown;
const char *token_uribl;
const char *token_verify;
const char *token_white;
const char *token_white_regex;
const char *token_yes;
const char *token_dkim_signer;
const char *token_dkim_from;
const char *token_signed_white;
const char *token_signed_black;
const char *token_require_signed;
const char *token_myhostname;

#ifndef HOST_NAME_MAX
    #define HOST_NAME_MAX 255
#endif
char myhostname[HOST_NAME_MAX+1];

pthread_mutex_t verifier_mutex;     // protect the verifier map
verify_map  verifiers;

pthread_mutex_t whitelister_mutex;  // protect the whitelisters map
whitelister_map whitelisters;

string_set  all_strings;    // owns all the strings, only modified by the config loader thread
const int maxlen = 1000;    // used for snprintf buffers
const int maxsmtp_age =  60;// smtp verify sockets older than this are ancient
const int maxauto_age = 600;// auto whitelister delay before flushing to file
extern int    NULL_SOCKET;
const time_t  ERROR_SMTP_SOCKET_TIME = 600; // number of seconds between attempts to open a socket to an smtp server


int SMTP::writer() {
    #ifdef VERIFY_DEBUG
        log("writer(%d) sees buffer with '%s'", buffer);
    #endif
    int rs = 0;
    if (!error) {
        int len = strlen(buffer);
        while (rs < len) {
            int ws = write(fd, buffer+rs, len-rs);
            if (ws > 0) {
                rs += ws;
            }
            else {
                // peer closed the socket!
                rs = 0;
                error = true;
                break;
            }
        }
    }
    #ifdef VERIFY_DEBUG
        log("writer(%d) sees error %d", (int)error);
    #endif
    return rs;
}


int SMTP::reader() {
    // read some bytes terminated by lf or end of buffer.
    // we may have a multi line response or part thereof in the buffer.
    #ifdef VERIFY_DEBUG
        log("reader(%d) sees error %d", (int)error);
    #endif
    if (error) return 0;
    int len = maxlen-1; // room for null terminator
    while (pending < len) {
        int ws = read(fd, buffer+pending, len-pending);
        if (ws > 0) {
            pending += ws;
            if (buffer[pending-1] == '\n') break;
        }
        else {
            // peer closed the socket!
            pending = 0;
            error = true;
            break;
        }
    }
    buffer[pending] = '\0';
    #ifdef VERIFY_DEBUG
        log("reader(%d) sees buffer with '%s'", buffer);
    #endif
    return pending;
}


int SMTP::read_line() {
    char *lf = strchr(buffer, '\n');
    if (!lf) {
        reader();  // get a lf
        lf = strchr(buffer, '\n');
        if (!lf) lf = buffer + pending - 1;
    }
    return (lf-buffer)+1;   // number of bytes in this line
}


void SMTP::flush_line(int r) {
    if (pending > r) memmove(buffer, buffer+r, pending-r);
    pending -= r;
}


int SMTP::read_response() {
    pending = 0;
    buffer[pending] = '\0';
    while (true) {
        int r = read_line();
        log("verify::read_response(%d) sees line with '%s'", buffer);
        if (r == 0) return 0;   // failed to read any bytes
        if ((r > 4) && (buffer[3] == '-')) {
            flush_line(r);
            continue;
        }
        return atoi(buffer);
    }
    return 0;
}


int SMTP::cmd(const char *c) {
    if (c) {
        init();
        append(c);
    }
    append("\r\n");
    writer();
    return read_response();
}


int SMTP::helo() {
    if (read_response() != 220) return 0;
    init();
    append("HELO ");
    append(token_myhostname);
    return cmd(NULL);
}


int SMTP::rset() {
    efrom[0] = '\0';
    return cmd("RSET");
}


int SMTP::from(const char *f) {
    // the mail from address was originally passed in from sendmail enclosed in
    // <>. to_lower_string() removed the <> and converted the rest to lowercase,
    // except in the case of an empty return path, which was left as the two
    // character string <>.
    if (strncmp(efrom, f, maxlen)) {
        strncpy(efrom, f, maxlen);
        efrom[maxlen-1] = '\0';     // ensure null termination
        init();
        append("MAIL FROM:<");
        if (*f != '<') append(f);
        append(">");
        return cmd(NULL);
    }
    return 250; // pretend it worked
}


int SMTP::rcpt(const char *t) {
    init();
    append("RCPT TO:<");
    append(t);
    append(">");
    return cmd(NULL);
}


int SMTP::quit() {
    return cmd("QUIT");
}


void SMTP::closefd() {
    shutdown(fd, SHUT_RDWR);
    close(fd);
}


void SMTP::log(const char *m, int v) {
    char buf[maxlen];
    snprintf(buf, maxlen, m, get_fd(), v);
    my_syslog(queueid, buf);
}


void SMTP::log(const char *m, const char *v) {
    char buf[maxlen];
    snprintf(buf, maxlen, m, get_fd(), v);
    my_syslog(queueid, buf);
 }


////////////////////////////////////////////////
// smtp verifier so backup mx machines can see the valid users
//
VERIFY::VERIFY(const char *h) {
    host     = h;
    last_err = 0;
    pthread_mutex_init(&mutex, 0);
}


void VERIFY::log(const char *m, const char *q, const char *v) {
    char buf[maxlen];
    snprintf(buf, maxlen, m, v, host);
    my_syslog(q, buf);
}


void VERIFY::closer() {
    bool ok = true;
    while (ok) {
        SMTP *conn = NULL;
        pthread_mutex_lock(&mutex);
            if (connections.empty()) {
                ok = false;
            }
            else {
                conn = connections.front();
                time_t now = time(NULL);
                if ((now - conn->get_stamp()) > maxsmtp_age) {
                    // this connection is ancient, remove it
                    connections.pop_front();
                }
                else {
                    ok   = false;
                    conn = NULL;
                }
            }
        pthread_mutex_unlock(&mutex);
        // avoid doing this work inside the mutex lock
        if (conn) {
            #ifdef VERIFY_DEBUG
                conn->log("closer(%d) closes ancient socket %s", "");
            #endif
            delete conn;
        }
    }
}


SMTP* VERIFY::get_connection(const char *queueid) {
    SMTP *conn = NULL;
    pthread_mutex_lock(&mutex);
        while (!connections.empty()) {
            conn = connections.front();
            time_t now = time(NULL);
            if ((now - conn->get_stamp()) > maxsmtp_age) {
                // this connection is ancient, remove it
                conn->log("verify::get_connection(%d) closes ancient socket %s", "");
                connections.pop_front();
                delete conn;
                conn = NULL;
            }
            else {
                conn->set_id(queueid);
                connections.pop_front();
                conn->log("verify::get_connection(%d) from cache %s", "");
                break;
            }
        }
    pthread_mutex_unlock(&mutex);
    if (conn) {
        int rc = conn->rset();
        conn->log("verify::getconnection(%d) rset sees %d", rc);
        if (rc == 250) return conn;
        delete conn;
        // old connection from cache was unusable, fall thru and make a new one
    }
    int sock = NULL_SOCKET;
    if ((time(NULL) - last_err) > ERROR_SMTP_SOCKET_TIME) {
        // nothing recent, maybe this time it will work
        hostent *h = gethostbyname(host);
        if (h) {
            sockaddr_in server;
            server.sin_family = h->h_addrtype;
            server.sin_port   = htons(25);
            memcpy(&server.sin_addr, h->h_addr_list[0], h->h_length);
            sock = socket(PF_INET, SOCK_STREAM, 0);
            if (sock != NULL_SOCKET) {
                bool rc = (connect(sock, (sockaddr *)&server, sizeof(server)) == 0);
                if (!rc) {
                    shutdown(sock, SHUT_RDWR);
                    close(sock);
                    sock = NULL_SOCKET;
                    last_err = time(NULL);
                }
            }
            else last_err = time(NULL);
        }
        else last_err = time(NULL);
    }
    if (sock != NULL_SOCKET) {
        struct timeval tv;
        tv.tv_sec = 15;
        tv.tv_usec = 0;
        setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&tv, sizeof(struct timeval));
        setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(struct timeval));
        conn = new SMTP(sock);
        conn->set_id(queueid);
        conn->log("get_connection(%d) new socket %s", "");
        int rc = conn->helo();
        conn->log("verify::get_connection(%d) helo sees %d", rc);
        if (rc == 250) return conn;
        delete conn;
    }
    return NULL;
}


void VERIFY::put_connection(SMTP *conn) {
    if (conn->err()) {
        #ifdef VERIFY_DEBUG
            conn->log("put_socket(%d) with error %s", "");
        #endif
        delete conn;
    }
    else {
        #ifdef VERIFY_DEBUG
            conn->log("put_socket(%d) no error %s", "");
        #endif
        conn->now();
        pthread_mutex_lock(&mutex);
            connections.push_back(conn);
        pthread_mutex_unlock(&mutex);
    }
}


bool VERIFY::ok(const char *queueid, const char *from, const char *to) {
    if (host == token_myhostname) return true;
    SMTP *conn = get_connection(queueid);
    if (!conn) {
        log("unable to verify %s with %s due to socket errors", queueid, to);
        return true;    // cannot verify right now, we have socket errors
    }
    int rc;
    rc = conn->from(from);
    conn->log("verify::ok(%d) from sees %d", rc);
    if (rc != 250) {
        put_connection(conn);
        return (rc >= 500) ? false : true;
    }
    rc = conn->rcpt(to);
    conn->log("verify::ok(%d) rcpt sees %d", rc);
    put_connection(conn);
    return (rc >= 500) ? false : true;
}


////////////////////////////////////////////////
// setup a new smtp verify host
//
VERIFYP add_verify_host(const char *host);
VERIFYP add_verify_host(const char *host) {
    VERIFYP rc = NULL;
    pthread_mutex_lock(&verifier_mutex);
        verify_map::iterator i = verifiers.find(host);
        if (i == verifiers.end()) {
            rc = new VERIFY(host);
            verifiers[host] = rc;
        }
        else rc = (*i).second;
    pthread_mutex_unlock(&verifier_mutex);
    return rc;
}


////////////////////////////////////////////////
// thread to check for verify hosts with old sockets that we can close
//
void* verify_closer(void *arg) {
    while (true) {
        sleep(maxsmtp_age);
        pthread_mutex_lock(&verifier_mutex);
            for (verify_map::iterator i=verifiers.begin(); i!=verifiers.end(); i++) {
                VERIFYP v = (*i).second;
                v->closer();
            }
        pthread_mutex_unlock(&verifier_mutex);
    }
    return NULL;
}


////////////////////////////////////////////////
// automatic whitelister
//
WHITELISTER::WHITELISTER(const char *f, int d) {
    fn       = f;
    days     = d;
    pthread_mutex_init(&mutex, 0);
    need     = false;
    loaded   = time(NULL);
    merge();
}


void WHITELISTER::merge() {
    time_t now = time(NULL);
    ifstream ifs;
    ifs.open(fn);
    if (!ifs.fail()) {
        const int maxlen = 1000;
        char buf[maxlen];
        while (ifs.getline(buf, maxlen)) {
            char *p = strchr(buf, ' ');
            if (p) {
                *p = '\0';
                char   *who = strdup(buf);
                time_t when = atoi(p+1);
                if ((when == 0) || (when > now)) when = now;
                autowhite_sent::iterator i = rcpts.find(who);
                if (i == rcpts.end()) {
                    rcpts[who] = when;
                }
                else {
                    time_t wh = (*i).second;
                    if ((when == 1) || (when > wh)) (*i).second = when;
                    free(who);
                }
            }
        }
    }
    ifs.close();
}


void WHITELISTER::writer() {
    pthread_mutex_lock(&mutex);
        time_t limit = time(NULL) - days*86400;

        // check for manually modified autowhitelist file
        struct stat st;
        if (stat(fn, &st)) need = true; // file has disappeared
        else if (st.st_mtime > loaded) {
            // file has been manually updated, merge new entries
            merge();
            need = true;
        }

        // purge old entries
        for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end();) {
            time_t when = (*i).second;
            if (when < limit) {
                const char *who = (*i).first;
                free((void*)who);
                rcpts.erase(i++);
                need = true;
            }
            else i++;
        }

        if (need) {
            // dump the file
            ofstream ofs;
            ofs.open(fn);
            if (!ofs.fail()) {
                for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end(); i++) {
                    const char *who = (*i).first;
                    int  when = (*i).second;
                    if (!strchr(who, ' ')) {
                        ofs << who << " " << when << endl;
                    }
                }
            }
            ofs.close();
            need = false;
            loaded = time(NULL);    // update load time
        }
    pthread_mutex_unlock(&mutex);
}


void WHITELISTER::sent(const char *to) {
    // we take ownership of the string
    pthread_mutex_lock(&mutex);
        need = true;
        autowhite_sent::iterator i = rcpts.find(to);
        if (i == rcpts.end()) {
            rcpts[to] = time(NULL);
        }
        else {
            (*i).second = time(NULL);
            free((void*)to);
        }
    pthread_mutex_unlock(&mutex);
}


bool WHITELISTER::is_white(const char *from) {
    pthread_mutex_lock(&mutex);
        autowhite_sent::iterator i = rcpts.find(from);
        bool rc = (i != rcpts.end());
    pthread_mutex_unlock(&mutex);
    return rc;
}


////////////////////////////////////////////////
// setup a new auto whitelister file
//
WHITELISTERP add_whitelister_file(const char *fn, int days);
WHITELISTERP add_whitelister_file(const char *fn, int days) {
    WHITELISTERP rc = NULL;
    pthread_mutex_lock(&whitelister_mutex);
        whitelister_map::iterator i = whitelisters.find(fn);
        if (i == whitelisters.end()) {
            rc = new WHITELISTER(fn, days);
            whitelisters[fn] = rc;
        }
        else {
            rc = (*i).second;
            rc->set_days(days);
        }
    pthread_mutex_unlock(&whitelister_mutex);
    return rc;
}


////////////////////////////////////////////////
// thread to check for whitelister hosts with old sockets that we can close
//
void* whitelister_writer(void *arg) {
    while (true) {
        sleep(maxauto_age);
        pthread_mutex_lock(&whitelister_mutex);
            for (whitelister_map::iterator i=whitelisters.begin(); i!=whitelisters.end(); i++) {
                WHITELISTERP v = (*i).second;
                v->writer();
            }
        pthread_mutex_unlock(&whitelister_mutex);
    }
    return NULL;
}


DELAYWHITE::DELAYWHITE(const char *loto_, WHITELISTERP w_, CONTEXTP con_) {
    loto = loto_;
    w    = w_;
    con  = con_;
}


DKIM::DKIM(const char *action_, const char *signer_) {
    action = action_;
    signer = signer_;
}


DNSBL::DNSBL(const char *n, const char *s, const char *m) {
    name    = n;
    suffix  = s;
    message = m;
}


bool DNSBL::operator==(const DNSBL &rhs) {
    return (strcmp(name,    rhs.name)    == 0) &&
           (strcmp(suffix,  rhs.suffix)  == 0) &&
           (strcmp(message, rhs.message) == 0);
}


DNSWL::DNSWL(const char *n, const char *s, const int l) {
    name    = n;
    suffix  = s;
    level   = l;
}


bool DNSWL::operator==(const DNSWL &rhs) {
    return (strcmp(name,    rhs.name)    == 0) &&
           (strcmp(suffix,  rhs.suffix)  == 0) &&
           (level == rhs.level);
}


CONFIG::CONFIG() {
    reference_count    = 0;
    generation         = 0;
    load_time          = 0;
    default_context    = NULL;
}


CONFIG::~CONFIG() {
    if (debug_syslog) {
        char buf[maxlen];
        snprintf(buf, sizeof(buf), "freeing memory for old configuration generation %d", generation);
        my_syslog(buf);
    }
    for (context_list::iterator i=contexts.begin(); i!=contexts.end(); i++) {
        CONTEXT *c = *i;
        delete c;
    }
}


void CONFIG::add_context(CONTEXTP con) {
    contexts.push_back(con);
    if (!default_context && !con->get_parent()) {
        // first global context
        default_context = con;
    }
}


void CONFIG::add_to(const char *to, CONTEXTP con) {
    context_map::iterator i = env_to.find(to);
    if (i != env_to.end()) {
        CONTEXTP  c = (*i).second;
        if ((c != con) && (c != con->get_parent())) {
            if (debug_syslog) {
                char oldname[maxlen];
                char newname[maxlen];
                const char *oldn = c->get_full_name(oldname, maxlen);
                const char *newn = con->get_full_name(newname, maxlen);
                char buf[maxlen*3];
                snprintf(buf, maxlen*3, "both %s and %s claim envelope to %s, the second one wins", oldn, newn, to);
                my_syslog(buf);
            }
        }
    }
    env_to[to] = con;
}


CONTEXTP CONFIG::find_context(const char *to) {
    context_map::iterator i = env_to.find(to);
    if (i != env_to.end()) return (*i).second;      // found user@domain key
    const char *x = strchr(to, '@');
    if (x) {
        x++;
        i = env_to.find(x);
        if (i != env_to.end()) return (*i).second;  // found domain key
        size_t len = x - to;
        char user[len+1];
        memcpy(user, to, len);
        user[len] = '\0';
        i = env_to.find(user);
        if (i != env_to.end()) return (*i).second;  // found user@ key
    }
    return default_context;
}


void CONFIG::dump() {
    bool spamass = false;
    if (default_context) default_context->dump(true, spamass);
    for (context_list::iterator i=contexts.begin(); i!=contexts.end(); i++) {
        CONTEXTP c = *i;
        CONTEXTP p = c->get_parent();
        if (!p && (c != default_context)) c->dump(false, spamass);
    }
    char buf[maxlen];
    for (context_map::iterator i=env_to.begin(); i!=env_to.end(); i++) {
        const char *to = (*i).first;
        CONTEXTP   con = (*i).second;
        printf("// envelope to %s \t-> context %s \n", to, con->get_full_name(buf,maxlen));
    }
    if (spamass && (spamc == spamc_empty)) {
        printf("// *** warning - spamassassin filtering requested, but spamc not found by autoconf.\n");
    }
}


CONTEXT::CONTEXT(CONTEXTP parent_, const char *name_) {
    parent                  = parent_;
    name                    = name_;
    verify_host             = NULL;
    verifier                = NULL;
    generic_regx            = NULL;
    generic_message         = NULL;
    white_regx              = NULL;
    autowhite_file          = NULL;
    whitelister             = NULL;
    env_from_default        = (parent) ? token_inherit : token_unknown;
    content_filtering       = (parent) ? parent->content_filtering : false;
    content_suffix          = NULL;
    content_message         = NULL;
    uribl_suffix            = NULL;
    uribl_message           = NULL;
    host_limit              = (parent) ? parent->host_limit  : 0;
    host_limit_message      = NULL;
    host_random             = (parent) ? parent->host_random : false;
    tag_limit               = (parent) ? parent->tag_limit   : 0;
    tag_limit_message       = NULL;
    spamassassin_limit      = (parent) ? parent->spamassassin_limit : 0;
    require_match           = (parent) ? parent->require_match      : false;
    require_rdns            = (parent) ? parent->require_rdns       : false;
    dcc_greylist            = (parent) ? parent->dcc_greylist       : false;
    dcc_bulk_threshold      = (parent) ? parent->dcc_bulk_threshold : 0;
    dnsbl_list_parsed       = false;
    dnswl_list_parsed       = false;
    default_rate_limit      = 36000;        // 10 per second
    default_address_limit   = 10;
    daily_rate_multiple     = 3;
    daily_address_multiple  = 3;
}


CONTEXT::~CONTEXT() {
    for (dkimp_map::iterator i=dkim_from_names.begin(); i!=dkim_from_names.end(); i++) {
        DKIMP d = (*i).second;
        // delete the underlying DKIM objects.
        delete d;
    }
    for (dnsblp_map::iterator i=dnsbl_names.begin(); i!=dnsbl_names.end(); i++) {
        DNSBLP d = (*i).second;
        // delete the underlying DNSBL objects.
        delete d;
    }
    if (generic_regx) regfree(&generic_pattern);
    if (white_regx)   regfree(&white_pattern);
}


bool CONTEXT::is_parent(CONTEXTP p) {
    if (p == parent) return true;
    if (!parent) return false;
    return parent->is_parent(p);
}


const char *CONTEXT::get_full_name(char *buffer, int size) {
    if (!parent) return name;
    char buf[maxlen];
    snprintf(buffer, size, "%s.%s", parent->get_full_name(buf, maxlen), name);
    return buffer;
}


bool CONTEXT::set_white(const char *regx)
{
    int rc = 0;
    if (white_regx) regfree(&white_pattern);
    white_regx = regx;
    if (white_regx) {
        rc = regcomp(&white_pattern, regx, REG_NOSUB | REG_ICASE | REG_EXTENDED);
    }
    return rc;  // true iff bad pattern
}


bool CONTEXT::white_match(const char *from)
{
    return (from       &&
            white_regx &&
            (0 == regexec(&white_pattern, from, 0, NULL, 0)));
}


bool CONTEXT::set_generic(const char *regx, const char *msg)
{
    int rc = 0;
    if (generic_regx) regfree(&generic_pattern);
    generic_regx    = regx;
    generic_message = msg;
    if (generic_regx) {
        rc = regcomp(&generic_pattern, regx, REG_NOSUB | REG_ICASE | REG_EXTENDED);
    }
    return rc;  // true iff bad pattern
}


const char *CONTEXT::generic_match(const char *client)
{
    if (!client) return NULL;   // allow missing _ macro, which will disable generic checking
    if (parent && !generic_regx) return parent->generic_match(client);
    if (!generic_regx)           return NULL;
    if (0 == regexec(&generic_pattern, client, 0, NULL, 0)) {
        return generic_message;
    }
    return NULL;
}


bool CONTEXT::cover_env_to(const char *to) {
    const char *x = strchr(to, '@');
    if (x) x++;
    else   x = to;
    if (*x == '\0') return true;                // always allow covering addresses with no domain name, eg abuse@
    if (!parent && env_to.empty()) return true; // empty env_to at global level covers everything
    string_set::iterator i = env_to.find(x);
    if (i != env_to.end()) return true;         // we cover the entire domain
    if (x != to) {
        i = env_to.find(to);
        if (i != env_to.end()) return true;     // we cover the specific email address
    }
    return false;
}


VERIFYP CONTEXT::find_verify(const char *to) {
    if (verifier && (verify_host != token_myhostname) && cover_env_to(to))
        return verifier;
    else if (parent)
        return parent->find_verify(to);
    else
        return NULL;
}


WHITELISTERP CONTEXT::find_autowhite(const char *from, const char *to) {
    if (whitelister && cover_env_to(to) && !cover_env_to(from))
        return whitelister;
    else if (parent)
        return parent->find_autowhite(from, to);
    else
        return NULL;
}


int CONTEXT::find_rate_limit(const char *user) {
    if (rcpt_per_hour.empty()) return default_rate_limit;
    rates::iterator i = rcpt_per_hour.find(user);       // look for authen id, or sender user@email limiting
    if (i != rcpt_per_hour.end()) return (*i).second;   // found authen id, or user@email limiting
    const char *f = strchr(user, '@');
    if (!f) return default_rate_limit;
    i = rcpt_per_hour.find(f);                          // look for @domain limiting
    if (i != rcpt_per_hour.end()) return (*i).second;   // found @domain limiting
    return default_rate_limit;
}


int CONTEXT::find_address_limit(const char *user) {
    if (addresses_per_hour.empty()) return default_address_limit;
    rates::iterator i = addresses_per_hour.find(user);      // look for authen id, or sender user@email limiting
    if (i != addresses_per_hour.end()) return (*i).second;  // found authen id, or user@email limiting
    const char *f = strchr(user, '@');
    if (!f) return default_address_limit;
    i = addresses_per_hour.find(f);                         // look for @domain limiting
    if (i != addresses_per_hour.end()) return (*i).second;  // found @domain limiting
    return default_address_limit;
}


bool CONTEXT::is_unauthenticated_limited(const char *user) {
    rates::iterator i = rcpt_per_hour.find(user);       // look for sender user@email limiting
    if (i != rcpt_per_hour.end()) return true;          // found user@email limiting
    const char *f = strchr(user, '@');
    if (!f) return false;
    i = rcpt_per_hour.find(f);                  // look for sender @domain limiting
    return (i != rcpt_per_hour.end());          // found @domain limiting
}


const char *CONTEXT::find_from(const char *from, bool update_white, const char *queueid) {
    WHITELISTERP w = whitelister;
    CONTEXTP     p = parent;
    while (!w && p) {
        w = p->whitelister;
        p = p->parent;
    }
    if (w && w->is_white(from)) {
        if (update_white && queueid) {
            // update senders timestamp to extend the whitelisting period
            if (debug_syslog > 1) {
                char buf[maxlen];
                char msg[maxlen];
                snprintf(msg, sizeof(msg), "%s: extend whitelist reply from <%s> in context %s", queueid, from, get_full_name(buf,maxlen));
                my_syslog(msg);
            }
            w->sent(strdup(from));
        }
        return token_white;
    }
    const char *rc = env_from_default;
    string_map::iterator i = env_from.find(from);
    if (i != env_from.end()) rc = (*i).second;  // found user@domain key
    else {
        const char *x = strchr(from, '@');
        if (x) {
            char buf[200];
            x++;
            i = env_from.find(x);
            size_t n = x - from;    // length of user name plus @
            if (i != env_from.end()) rc = (*i).second;  // found domain key
            else if (n < sizeof(buf)) {
                // we only test reasonably short user names, since we need
                // to copy them to a buffer to avoid a dup/free cycle on every
                // test here.
                strncpy(buf, from, n);
                buf[n] = '\0';
                i = env_from.find(buf);
                if (i != env_from.end()) rc = (*i).second;  // found user@ key
            }
        }
    }
    if ((rc == token_inherit) || (rc == token_unknown)) {
        bool ok = white_match(from);
        if (ok) rc = token_white;
    }
    if ((rc == token_inherit) && parent) return parent->find_from(from);
    return (rc == token_inherit) ? token_unknown : rc;
}


CONTEXTP CONTEXT::find_context(const char *from) {
    context_map::iterator i = env_from_context.find(from);
    if (i != env_from_context.end()) return (*i).second;        // found user@domain key
    const char *x = strchr(from, '@');
    if (x) {
        char buf[200];
        x++;
        i = env_from_context.find(x);
        size_t n = x - from;    // length of user name plus @
        if (i != env_from_context.end()) return (*i).second;    // found domain key
        else if (n < sizeof(buf)) {
            // we only test reasonably short user names, since we need
            // to copy them to a buffer to avoid a dup/free cycle on every
            // test here.
            strncpy(buf, from, n);
            buf[n] = '\0';
            i = env_from_context.find(buf);
            if (i != env_from_context.end()) return (*i).second;    // found user@ key
        }
    }
    return this;
}


CONTEXTP CONTEXT::find_from_context_name(const char *name) {
    context_map::iterator i = children.find(name);
    if (i != children.end()) return (*i).second;
    return NULL;
}


const char *CONTEXT::find_dkim_signer(const char *name) {
    if (!name) return NULL;
    string_map::iterator i = dkim_signer_names.find(name);
    if (i != dkim_signer_names.end()) return (*i).second;
    if (parent) return parent->find_dkim_signer(name);
    return NULL;
}


DKIMP CONTEXT::find_dkim_from(const char *name) {
    if (!name) return NULL;
    dkimp_map::iterator i = dkim_from_names.find(name);
    if (i != dkim_from_names.end()) return (*i).second;
    if (parent) return parent->find_dkim_from(name);
    return NULL;
}


DNSBLP CONTEXT::find_dnsbl(const char *name) {
    dnsblp_map::iterator i = dnsbl_names.find(name);
    if (i != dnsbl_names.end()) return (*i).second;
    if (parent) return parent->find_dnsbl(name);
    return NULL;
}


DNSWLP CONTEXT::find_dnswl(const char *name) {
    dnswlp_map::iterator i = dnswl_names.find(name);
    if (i != dnswl_names.end()) return (*i).second;
    if (parent) return parent->find_dnswl(name);
    return NULL;
}


const char* CONTEXT::get_content_suffix() {
    if (!content_suffix && parent) return parent->get_content_suffix();
    return content_suffix;
}


const char* CONTEXT::get_uribl_suffix() {
    if (!uribl_suffix && parent) return parent->get_uribl_suffix();
    return uribl_suffix;
}


const char* CONTEXT::get_content_message() {
    if (!content_message && parent) return parent->get_content_message();
    return content_message;
}


const char* CONTEXT::get_uribl_message() {
    if (!uribl_message && parent) return parent->get_uribl_message();
    return uribl_message;
}


string_set& CONTEXT::get_content_host_ignore() {
    if (content_host_ignore.empty() && parent) return parent->get_content_host_ignore();
    return content_host_ignore;
}


string_set& CONTEXT::get_content_tlds() {
    if (content_tlds.empty() && parent) return parent->get_content_tlds();
    return content_tlds;
}


string_set& CONTEXT::get_content_tldwilds() {
    if (content_tldwilds.empty() && parent) return parent->get_content_tldwilds();
    return content_tldwilds;
}


string_set& CONTEXT::get_content_tldnots() {
    if (content_tldnots.empty() && parent) return parent->get_content_tldnots();
    return content_tldnots;
}


string_set& CONTEXT::get_html_tags() {
    if (html_tags.empty() && parent) return parent->get_html_tags();
    return html_tags;
}


dnsblp_list& CONTEXT::get_dnsbl_list() {
    if (!dnsbl_list_parsed && parent) return parent->get_dnsbl_list();
    return dnsbl_list;
}


dnswlp_list& CONTEXT::get_dnswl_list() {
    if (!dnswl_list_parsed && parent) return parent->get_dnswl_list();
    return dnswl_list;
}


void CONTEXT::log(const char *queueid, const char *msg, const char *v) {
    if (debug_syslog > 1) {
        char buf[maxlen];
        snprintf(buf, maxlen, msg, v);
        my_syslog(queueid, buf);
    }
}


bool CONTEXT::in_signing_set(const char *s, const char *signers) {
    size_t n = strlen(s);
    const char *p = signers;
    do {
        if ((strncasecmp(p, s, n) == 0) && ((p[n] == '\0') || (p[n] == ','))) return true;
        p = strchr(p, ',');
        if (!p) return false;
        p++;
    } while (true);
}


bool CONTEXT::resolve_spf(const char *from, uint32_t ip, mlfiPriv *priv, int level)
{
    // ip is in host order
    char buf[maxlen];
    log(priv->queueid, "looking for %s txt record", from);
    dns_interface(*priv, from, ns_t_txt, false, NULL, buf, maxlen);
    if (*buf) {
        log(priv->queueid, "found txt record %s", buf);
        char *p = buf;
        char *e = p + strlen(p);    // point to trailing null
        while ((p = strstr(p, " ip4:"))) {
            p += 5;
            char *b = strchr(p, ' ');
            if (b) *b = '\0';
            char *s = strchr(p, '/');
            if (s) *s = '\0';
            in_addr ipx;
            if (inet_aton(p, &ipx)) {
                uint32_t ipy = ntohl(ipx.s_addr);
                int mask = (s) ? atoi(s+1) : 32;
                if ((mask >= 16) && (mask <= 32)) {
                    uint32_t low = (1 << (32-mask)) - 1;
                    ipy &= low ^ 0xffffffff;
                    if ((ipy <= ip) && (ip <= ipy + low)) {
                        if (s) *s = '/';
                        log(priv->queueid, "match %s", p);
                        return true;
                    }
                }
            }
            if (b) *b = ' ';
            if (s) *s = '/';
            p = (b) ? b : e;
        }
        p = buf;
        while ((level < 5) && (p = strstr(p, " include:"))) {
            p += 9;
            char *b = strchr(p, ' ');
            if (b) *b = '\0';
            if (resolve_spf(p, ip, priv, level+1)) return true;
            if (b) *b = ' ';
            p = (b) ? b : e;
        }
    }
    return false;
}


const char *CONTEXT::acceptable_content(recorder &memory, int score, int bulk, const char *queueid, string_set &signers, const char *from, mlfiPriv *priv, string& msg) {
    DKIMP dk = find_dkim_from(from);

    for (string_set::iterator s=signers.begin(); s!=signers.end(); s++) {
        const char *st = find_dkim_signer(*s);
        // signed by a white listed signer
        if (st == token_white) {
            log(queueid, "whitelisted dkim signer %s", *s);
            return token_white;
        }
    }

    if (dk) {
        const char *st = dk->action;
        for (string_set::iterator s=signers.begin(); s!=signers.end(); s++) {
            // signed by a white listed signer
            if ((st == token_signed_white)   && in_signing_set(*s,dk->signer)) {
                log(queueid, "whitelisted dkim signer %s", *s);
                return token_white;
            }
            // signed by the required signer
            if ((st == token_require_signed) && in_signing_set(*s,dk->signer)) {
                log(queueid, "required dkim signer %s", *s);
                return token_white;
            }
            // signed by a black listed signer
            if ((st == token_signed_black)   && in_signing_set(*s,dk->signer)) {
                char buf[maxlen];
                snprintf(buf, sizeof(buf), "Mail rejected - dkim signed by %s", *s);
                msg = string(buf);
                return token_black;
            }
        }
        if (st == token_signed_white) {
            // not signed by a white signer, but maybe passes strong spf check
            if (resolve_spf(from, ntohl(priv->ip), priv)) {
                log(queueid, "spf pass for %s rather than whitelisted dkim signer", from);
                return token_white;
            }
        }
        if (st == token_require_signed) {
            // not signed by a required signer, but maybe passes strong spf check
            // only check spf if the list of required signers is not a single blank.
            if (strcmp(dk->signer, " ") && resolve_spf(from, ntohl(priv->ip), priv)) {
                log(queueid, "spf pass for %s rather than required dkim signer", from);
                return token_white;
            }
            char buf[maxlen];
            snprintf(buf, sizeof(buf), "Mail rejected - not dkim signed by %s", dk->signer);
            msg = string(buf);
            return token_black;
        }
    }

    for (string_set::iterator s=signers.begin(); s!=signers.end(); s++) {
        const char *st = find_dkim_signer(*s);
        // signed by a black listed signer
        if (st == token_black) {
            char buf[maxlen];
            snprintf(buf, sizeof(buf), "Mail rejected - dkim signed by %s", *s);
            msg = string(buf);
            return token_black;
        }
    }

    if (spamassassin_limit && (score > spamassassin_limit)) {
        char buf[maxlen];
        snprintf(buf, sizeof(buf), "Mail rejected - spam assassin score %d", score);
        msg = string(buf);
        return token_black;
    }
    if (dcc_bulk_threshold && (bulk >= dcc_bulk_threshold)) {
        char buf[maxlen];
        snprintf(buf, sizeof(buf), "Mail rejected - dcc score %d", bulk);
        msg = string(buf);
        return token_black;
    }
    if (memory.excessive_bad_tags(tag_limit)) {
        msg = string(tag_limit_message);
        return token_black;
    }
    if (!host_random && memory.excessive_hosts(host_limit)) {
        msg = string(host_limit_message);
        return token_black;
    }
    return token_unknown;
}


void CONTEXT::dump(bool isdefault, bool &spamass, int level) {
    char indent[maxlen];
    int i = min(maxlen-1, level*4);
    memset(indent, ' ', i);
    indent[i] = '\0';
    char buf[maxlen];
    const char *fullname = get_full_name(buf,maxlen);
    printf("%s context %s { \t// %s\n", indent, name, fullname);

    for (dnsblp_map::iterator i=dnsbl_names.begin(); i!=dnsbl_names.end(); i++) {
        const char *n = (*i).first;
        DNSBL &d = *(*i).second;
        printf("%s     dnsbl %s %s \"%s\"; \n", indent, n, d.suffix, d.message);
    }

    for (dnswlp_map::iterator i=dnswl_names.begin(); i!=dnswl_names.end(); i++) {
        const char *n = (*i).first;
        DNSWL &d = *(*i).second;
        printf("%s     dnswl %s %s %d; \n", indent, n, d.suffix, d.level);
    }

    {
        dnsblp_list dl = get_dnsbl_list();
        printf("%s     dnsbl_list", indent);
        for (dnsblp_list::iterator i=dl.begin(); i!=dl.end(); i++) {
            DNSBL &d = *(*i);
            printf(" %s", d.name);
        }
        printf("; \n");
        printf("%s     require_rdns  %s; \n", indent, (require_rdns)  ? "yes" : "no");
    }

    {
        dnswlp_list dl = get_dnswl_list();
        printf("%s     dnswl_list", indent);
        for (dnswlp_list::iterator i=dl.begin(); i!=dl.end(); i++) {
            DNSWL &d = *(*i);
            printf(" %s", d.name);
        }
        printf("; \n");
    }

    if (content_filtering) {
        printf("%s     content on { \n", indent);
        printf("%s         dkim_signer { \n", indent);
        for (string_map::iterator i=dkim_signer_names.begin(); i!=dkim_signer_names.end(); i++) {
            const char *n = (*i).first;
            const char *a = (*i).second;
            printf("%s             %s %s; \n", indent, n, a);
        }
        printf("%s         }; \n", indent);
        printf("%s         dkim_from { \n", indent);
        for (dkimp_map::iterator i=dkim_from_names.begin(); i!=dkim_from_names.end(); i++) {
            const char *n = (*i).first;
            DKIM &d = *(*i).second;
            printf("%s             %s %s \"%s\"; \n", indent, n, d.action, d.signer);
        }
        printf("%s         }; \n", indent);
        if (content_suffix) {
            printf("%s         filter %s \"%s\"; \n", indent, content_suffix, content_message);
        }
        if (uribl_suffix) {
            printf("%s         uribl %s \"%s\"; \n", indent, uribl_suffix, uribl_message);
        }
        if (!content_host_ignore.empty()) {
            printf("%s         ignore { \n", indent);
            for (string_set::iterator i=content_host_ignore.begin(); i!=content_host_ignore.end(); i++) {
                printf("%s             %s; \n", indent, *i);
            }
            printf("%s         }; \n", indent);
        }
        if (!content_tlds.empty() || !content_tldwilds.empty() || !content_tldnots.empty()) {
            printf("%s         tld { \n", indent);
            printf("%s             ", indent);
            for (string_set::iterator i=content_tlds.begin(); i!=content_tlds.end(); i++) {
                printf("%s; ", *i);
            }
            for (string_set::iterator i=content_tldwilds.begin(); i!=content_tldwilds.end(); i++) {
                printf("*.%s; ", *i);
            }
            for (string_set::iterator i=content_tldnots.begin(); i!=content_tldnots.end(); i++) {
                printf("!%s; ", *i);
            }
            printf("\n%s         }; \n", indent);
        }
        if (!html_tags.empty()) {
            printf("%s         html_tags { \n", indent);
            printf("%s             ", indent);
            for (string_set::iterator i=html_tags.begin(); i!=html_tags.end(); i++) {
                printf("%s; ", *i);
            }
            printf("\n%s         }; \n", indent);
        }
        if (host_limit_message) {
            printf("%s         host_limit on %d \"%s\"; \n", indent, host_limit, host_limit_message);
        }
        else if (host_random) {
            printf("%s         host_limit soft %d; \n", indent, host_limit);
        }
        else {
            printf("%s         host_limit off; \n", indent);
        }
        if (tag_limit_message) {
            printf("%s         html_limit on %d \"%s\"; \n", indent, tag_limit, tag_limit_message);
        }
        else {
            printf("%s         html_limit off; \n", indent);
        }
        printf("%s         spamassassin %d; \n", indent, spamassassin_limit);
        printf("%s         require_match %s; \n", indent, (require_match) ? "yes" : "no");
        printf("%s         dcc_greylist  %s; \n", indent, (dcc_greylist)  ? "yes" : "no");
        if (dcc_bulk_threshold == 0)            printf("%s         dcc_bulk_threshold off; \n", indent);
        else if (dcc_bulk_threshold >= dccbulk) printf("%s         dcc_bulk_threshold many; \n", indent);
        else                                    printf("%s         dcc_bulk_threshold %d; \n", indent, dcc_bulk_threshold);
        printf("%s     }; \n", indent);
        spamass |= (spamassassin_limit != 0);
        }
    else {
        printf("%s     content off {}; \n", indent);
    }

    printf("%s     env_to { \t// %s\n", indent, fullname);
    for (string_set::iterator i=env_to.begin(); i!=env_to.end(); i++) {
        printf("%s         %s; \n", indent, *i);
    }
    printf("%s     }; \n", indent);

    if (verify_host) {
        printf("%s     verify %s; \n", indent, verify_host);
    }

    if (generic_regx) {
        printf("%s     generic \"%s\"  \n", indent, generic_regx);
        printf("%s             \"%s\"; \n", indent, generic_message);
    }

    if (white_regx) {
        printf("%s     white_regex \"%s\"; \n", indent, white_regx);
    }

    if (autowhite_file && whitelister) {
        printf("%s     autowhite %d %s; \n", indent, whitelister->get_days(), autowhite_file);
    }

    for (context_map::iterator i=children.begin(); i!=children.end(); i++) {
        CONTEXTP c = (*i).second;
        c->dump(false, spamass, level+1);
    }

    printf("%s     env_from %s { \t// %s\n", indent, env_from_default, fullname);
    if (!env_from.empty()) {
        printf("%s     // white/black/unknown \n", indent);
        for (string_map::iterator i=env_from.begin(); i!=env_from.end(); i++) {
            const char *f = (*i).first;
            const char *t = (*i).second;
            printf("%s         %s \t%s; \n", indent, f, t);
        }
    }
    if (!env_from_context.empty()) {
        printf("%s     // child contexts \n", indent);
        for (context_map::iterator j=env_from_context.begin(); j!=env_from_context.end(); j++) {
            const char    *f = (*j).first;
            CONTEXTP t = (*j).second;
            printf("%s         %s \t%s; \n", indent, f, t->name);
        }
    }
    printf("%s     }; \n", indent);

    if (isdefault) {
        printf("%s     rate_limit %d %d %d %d { \n", indent, default_rate_limit, daily_rate_multiple, default_address_limit, daily_address_multiple);
        for (rates::iterator j=rcpt_per_hour.begin(); j!=rcpt_per_hour.end(); j++) {
            const char     *u = (*j).first;
            int             l = (*j).second;
            rates::iterator k = addresses_per_hour.find(u);
            int             a = (k==addresses_per_hour.end()) ? default_address_limit : (*k).second;
            printf("%s         \"%s\" \t%d %d; \n", indent, u, l, a);
        }
        printf("%s     }; \n", indent);
    }

    printf("%s }; \n", indent);
}


////////////////////////////////////////////////
// helper to discard the strings held by a string_set
//
void discard(string_set &s) {
    for (string_set::iterator i=s.begin(); i!=s.end(); i++) {
        free((void*)*i);
    }
    s.clear();
}


////////////////////////////////////////////////
// helper to register a string in a string set
//
const char* register_string(string_set &s, const char *name) {
    string_set::iterator i = s.find(name);
    if (i != s.end()) return *i;
    char *x = strdup(name);
    s.insert(x);
    return x;
}


////////////////////////////////////////////////
// register a global string
//
const char* register_string(const char *name) {
    return register_string(all_strings, name);
}


////////////////////////////////////////////////
// clear all global strings, helper for valgrind checking
//
void clear_strings() {
    discard(all_strings);
}


////////////////////////////////////////////////
//
bool tsa(TOKEN &tok, const char *token);
bool tsa(TOKEN &tok, const char *token) {
    const char *have = tok.next();
    if (have == token) return true;
    tok.token_error(token, have);
    return false;
}


////////////////////////////////////////////////
//
bool parse_dnsbl(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnsbl(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *name = tok.next();
    const char *suf  = tok.next();
    const char *msg  = tok.next();
    if (!tsa(tok, token_semi)) return false;
    DNSBLP dnsnew = new DNSBL(name, suf, msg);
    DNSBLP dnsold = me.find_dnsbl(name);
    if (dnsold && (*dnsold == *dnsnew)) {
        // duplicate redefinition, ignore it
        delete dnsnew;
        return true;
    }
    me.add_dnsbl(name, dnsnew);
    return true;
}


////////////////////////////////////////////////
//
bool parse_dnswl(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnswl(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *name = tok.next();
    const char *suf  = tok.next();
    const int   lev  = tok.nextint();
    if (!tsa(tok, token_semi)) return false;
    DNSWLP dnsnew = new DNSWL(name, suf, lev);
    DNSWLP dnsold = me.find_dnswl(name);
    if (dnsold && (*dnsold == *dnsnew)) {
        // duplicate redefinition, ignore it
        delete dnsnew;
        return true;
    }
    me.add_dnswl(name, dnsnew);
    return true;
}


////////////////////////////////////////////////
//
bool parse_dnsbll(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnsbll(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_semi) break;
        DNSBLP dns = me.find_dnsbl(have);
        if (dns) {
            me.add_dnsbl(dns);
        }
        else {
            tok.token_error("dnsbl name", have);
            return false;
        }
    }
    me.set_dnsbll_parsed();
    return true;
}


////////////////////////////////////////////////
//
bool parse_dnswll(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnswll(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_semi) break;
        DNSWLP dns = me.find_dnswl(have);
        if (dns) {
            me.add_dnswl(dns);
        }
        else {
            tok.token_error("dnswl name", have);
            return false;
        }
    }
    me.set_dnswll_parsed();
    return true;
}


////////////////////////////////////////////////
//
bool parse_requirerdns(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_requirerdns(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *have = tok.next();
         if (have == token_yes) me.set_requirerdns(true);
    else if (have == token_no)  me.set_requirerdns(false);
    else {
        tok.token_error("yes/no", have);
        return false;
    }
    if (!tsa(tok, token_semi)) return false;
    return true;
}


////////////////////////////////////////////////
//
bool parse_dkim_signer(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dkim_signer(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        if (have == token_semi) {
            // optional separators
        }
        else {
            const char *signer = have;
            const char *action = tok.next();
            if ((action == token_white) || (action == token_black) || (action == token_unknown)) {
                me.add_dkim_signer(signer, action);
            }
            else {
                tok.token_error("white/black/unknown", action);
            }
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_dkim_from(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dkim_from(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        if (have == token_semi) {
            // optional separators
        }
        else {
            const char *from   = have;
            const char *action = tok.next();
            if ((action == token_signed_white) || (action == token_signed_black) || (action == token_require_signed)) {
                const char *signer = tok.next();
                if (!signer) break;
                else         me.add_dkim_from(from, action, signer);
            }
            else {
                tok.token_error("signed_white/signed_black/require_signed", action);
            }
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_content(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_content(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *setting = tok.next();
    if (setting == token_on) {
        me.set_content_filtering(true);
    }
    else if (setting == token_off) {
        me.set_content_filtering(false);
    }
    else {
        tok.token_error("on/off", setting);
        return false;
    }
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_filter) {
            const char *suffix = tok.next();
            const char *messag = tok.next();
            me.set_content_suffix(suffix);
            me.set_content_message(messag);
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_uribl) {
            const char *suffix = tok.next();
            const char *messag = tok.next();
            me.set_uribl_suffix(suffix);
            me.set_uribl_message(messag);
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_ignore) {
            if (!tsa(tok, token_lbrace)) return false;
            while (true) {
                if (!have) break;
                const char *have = tok.next();
                if (have == token_rbrace) break;  // done
                me.add_ignore(have);
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_tld) {
            if (!tsa(tok, token_lbrace)) return false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) break;  // done
                if (have == token_bang) {
                    have = tok.next();
                    if (!have) break;
                    if (have == token_rbrace) break;  // done
                    me.add_tldnot(have);
                }
                else if (have == token_asterisk) {
                    have = tok.next();
                    if (!have) break;
                    if (have == token_rbrace) break;  // done
                    if (have == token_period) {
                        have = tok.next();
                        if (!have) break;
                        if (have == token_rbrace) break;  // done
                        me.add_tldwild(have);
                    }
                }
                else me.add_tld(have);
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_html_tags) {
            if (!tsa(tok, token_lbrace)) return false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) {
                    break;  // done
                }
                else {
                    me.add_tag(have);                           // base version
                    char buf[200];
                    snprintf(buf, sizeof(buf), "/%s", have);
                    me.add_tag(register_string(buf));           // leading /
                    snprintf(buf, sizeof(buf), "%s/", have);
                    me.add_tag(register_string(buf));           // trailing /
                }
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_html_limit) {
            have = tok.next();
            if (have == token_on) {
                me.set_tag_limit(tok.nextint());
                me.set_tag_message(tok.next());
            }
            else if (have == token_off) {
                me.set_tag_limit(0);
                me.set_tag_message(NULL);
            }
            else {
                tok.token_error("on/off", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_host_limit) {
            have = tok.next();
            if (have == token_on) {
                me.set_host_limit(tok.nextint());
                me.set_host_message(tok.next());
                me.set_host_random(false);
            }
            else if (have == token_off) {
                me.set_host_limit(0);
                me.set_host_message(NULL);
                me.set_host_random(false);
            }
            else if (have == token_soft) {
                me.set_host_limit(tok.nextint());
                me.set_host_message(NULL);
                me.set_host_random(true);
            }
            else {
                tok.token_error("on/off/soft", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_spamassassin) {
            me.set_spamassassin_limit(tok.nextint());
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_require) {
            have = tok.next();
                 if (have == token_yes) me.set_require(true);
            else if (have == token_no)  me.set_require(false);
            else {
                tok.token_error("yes/no", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_dccgrey) {
            have = tok.next();
                 if (have == token_yes) me.set_grey(true);
            else if (have == token_no)  me.set_grey(false);
            else {
                tok.token_error("yes/no", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_dccbulk) {
            have = tok.next();
                 if (have == token_off) me.set_bulk(0);
            else if (have == token_many) me.set_bulk(dccbulk);
            else {
                char *e;
                long i = strtol(have, &e, 10);
                if (*e != '\0') {
                    tok.token_error("integer", have);
                    return false;
                }
                me.set_bulk((int)i);
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_dkim_signer) {
            if (!parse_dkim_signer(tok, dc, me)) return false;
        }
        else if (have == token_dkim_from) {
            if (!parse_dkim_from(tok, dc, me)) return false;
        }
        else if (have == token_rbrace) {
            break;  // done
        }
        else {
            tok.token_error("content keyword", have);
            return false;
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_envto(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_envto(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        if (have == token_semi) {
            // optional separators
        }
        else if (have == token_dccto) {
            const char *flavor = tok.next();
            if (!tsa(tok, token_lbrace)) return false;
            bool keeping = false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) break;
                if (have == flavor) {
                    keeping = true;
                    continue;
                }
                else if ((have == token_ok) || (have == token_ok2) || (have == token_many)) {
                    keeping = false;
                    continue;
                }
                if (have == token_envto) {
                    have = tok.next();
                    if (keeping) {
                        if (me.allow_env_to(have)) {
                            me.add_to(have);
                            dc.add_to(have, &me);
                        }
                    }
                }
              //else if (have == token_substitute) {
              //    if (tok.next() == token_mailhost) {
              //        have = tok.next();
              //        if (keeping) {
              //            if (me.allow_env_to(have)) {
              //                me.add_to(have);
              //                dc.add_to(have, &me);
              //            }
              //        }
              //    }
              //}
                tok.skipeol();
            }
        }
        else if (me.allow_env_to(have)) {
            me.add_to(have);
            dc.add_to(have, &me);
        }
        else {
            tok.token_error("user@ or user@domain.tld or domain.tld where domain.tld allowed by parent context", have);
            return false;
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_verify(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_verify(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *host = tok.next();
    if (!tsa(tok, token_semi)) return false;
    me.set_verify(host);
    me.set_verifier(add_verify_host(host));
    return true;
}


////////////////////////////////////////////////
//
bool parse_generic(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_generic(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *regx = tok.next();
    const char *msg  = tok.next();
    if (!tsa(tok, token_semi)) return false;
    if (me.set_generic(regx, msg)) {
        tok.token_error("invalid regular expression %s", regx, regx);
        return false;
    }
    return true;
}


////////////////////////////////////////////////
//
bool parse_white(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_white(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *regx = tok.next();
    if (!tsa(tok, token_semi)) return false;
    if (me.set_white(regx)) {
        tok.token_error("invalid regular expression %s", regx, regx);
        return false;
    }
    return true;
}


////////////////////////////////////////////////
//
bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    int days = tok.nextint();
    const char *fn = tok.next();
    if (!tsa(tok, token_semi)) return false;
    me.set_autowhite(fn);
    me.set_whitelister(add_whitelister_file(fn, days));
    return true;
}


////////////////////////////////////////////////
//
bool parse_envfrom(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_envfrom(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *st = tok.next();
    if ((st == token_black) || (st == token_white) || (st == token_unknown) || (st == token_inherit)) {
        me.set_from_default(st);
    }
    else {
        tok.push(st);
    }
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        if (have == token_semi) {
            // optional separators
        }
        else if (have == token_dccfrom) {
            if (!tsa(tok, token_lbrace)) return false;
            bool keeping = false;
            bool many = false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) break;
                if (have == token_ok) {
                    keeping = true;
                    many    = false;
                    continue;
                }
                else if (have == token_many) {
                    keeping = true;
                    many    = true;
                    continue;
                }
                else if (have == token_ok2) {
                    keeping = false;
                    continue;
                }
                if (have == token_envfrom) {
                    have = tok.next();
                    if (keeping) {
                        me.add_from(have, (many) ? token_black : token_white);
                    }
                }
                else if (have == token_substitute) {
                    if (tok.next() == token_mailhost) {
                        have = tok.next();
                        me.add_from(have, (many) ? token_black : token_white);
                    }
                }
                tok.skipeol();
            }
        }
        else {
            // may be a valid email address or domain name
            const char *st = tok.next();
            if ((st == token_white) || (st == token_black) || (st == token_unknown) || (st == token_inherit)) {
                me.add_from(have, st);
            }
            else {
                CONTEXTP con = me.find_from_context_name(st);
                if (con) {
                    me.add_from_context(have, con);
                }
                else {
                    tok.token_error("white/black/unknown/inherit or child context name", st);
                    return false;
                }
            }
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    me.set_default_rate_limit(tok.nextint());
    me.set_daily_rate_multiple(tok.nextint());
    me.set_default_address_limit(tok.nextint());
    me.set_daily_address_multiple(tok.nextint());
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        me.add_rate_limit(have, tok.nextint());
        me.add_address_limit(have, tok.nextint());
        if (!tsa(tok, token_semi)) return false;
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_context(TOKEN &tok, CONFIG &dc, CONTEXTP parent);
bool parse_context(TOKEN &tok, CONFIG &dc, CONTEXTP parent) {
    const char *name = tok.next();
    if (!tsa(tok, token_lbrace)) return false;
    CONTEXTP con = new CONTEXT(parent, name);

    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;  // done
        if (have == token_dnsbl) {
            if (!parse_dnsbl(tok, dc, *con)) return false;
        }
        else if (have == token_dnsbll) {
            if (!parse_dnsbll(tok, dc, *con)) return false;
        }
        else if (have == token_dnswl) {
            if (!parse_dnswl(tok, dc, *con)) return false;
        }
        else if (have == token_dnswll) {
            if (!parse_dnswll(tok, dc, *con)) return false;
        }
        else if (have == token_content) {
            if (!parse_content(tok, dc, *con)) return false;
        }
        else if (have == token_envto) {
            if (!parse_envto(tok, dc, *con)) return false;
        }
        else if (have == token_verify) {
            if (!parse_verify(tok, dc, *con)) return false;
        }
        else if (have == token_generic) {
            if (!parse_generic(tok, dc, *con)) return false;
        }
        else if (have == token_white_regex) {
            if (!parse_white(tok, dc, *con)) return false;
        }
        else if (have == token_autowhite) {
            if (!parse_autowhite(tok, dc, *con)) return false;
        }
        else if (have == token_envfrom) {
            if (!parse_envfrom(tok, dc, *con)) return false;
        }
        else if (have == token_dkim_signer) {
            if (!parse_dkim_signer(tok, dc, *con)) return false;
        }
        else if (have == token_dkim_from) {
            if (!parse_dkim_from(tok, dc, *con)) return false;
        }
        else if (have == token_rate) {
            if (parent || dc.default_context) tok.token_error("rate limit ignored in non default context");
            if (!parse_rate(tok, dc, *con)) return false;
        }
        else if (have == token_requirerdns) {
            if (!parse_requirerdns(tok, dc, *con)) return false;
        }
        else if (have == token_context) {
            if (!parse_context(tok, dc, con)) return false;
        }
        else {
            tok.token_error("context keyword", have);
            return false;
        }
    }

    if (!tsa(tok, token_semi)) {
        delete con;
        return false;
    }
    dc.add_context(con);
    if (parent) parent->add_context(con);
    return true;
}


////////////////////////////////////////////////
// parse a config file
//
bool load_conf(CONFIG &dc, const char *fn) {
    int count = 0;
    TOKEN tok(fn, &dc.config_files);
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_context) {
            if (!parse_context(tok, dc, NULL)) {
                tok.token_error("load_conf() failed to parse context");
                return false;
            }
            else count++;
        }
        else {
            tok.token_error(token_context, have);
            return false;
        }
    }
    tok.token_error("load_conf() found %d contexts in %s", count, fn);
    return (dc.default_context) ? true : false;
}


////////////////////////////////////////////////
// init the tokens
//
void token_init() {
    token_asterisk          = register_string("*");
    token_autowhite         = register_string("autowhite");
    token_bang              = register_string("!");
    token_black             = register_string("black");
    token_content           = register_string("content");
    token_context           = register_string("context");
    token_dccbulk           = register_string("dcc_bulk_threshold");
    token_dccfrom           = register_string("dcc_from");
    token_dccgrey           = register_string("dcc_greylist");
    token_dccto             = register_string("dcc_to");
    token_default           = register_string("default");
    token_dnsbl             = register_string("dnsbl");
    token_dnsbll            = register_string("dnsbl_list");
    token_dnswl             = register_string("dnswl");
    token_dnswll            = register_string("dnswl_list");
    token_envfrom           = register_string("env_from");
    token_envto             = register_string("env_to");
    token_filter            = register_string("filter");
    token_generic           = register_string("generic");
    token_host_limit        = register_string("host_limit");
    token_html_limit        = register_string("html_limit");
    token_html_tags         = register_string("html_tags");
    token_ignore            = register_string("ignore");
    token_include           = register_string("include");
    token_inherit           = register_string("inherit");
    token_lbrace            = register_string("{");
    token_mailhost          = register_string("mail_host");
    token_many              = register_string("many");
    token_no                = register_string("no");
    token_off               = register_string("off");
    token_ok                = register_string("ok");
    token_ok2               = register_string("ok2");
    token_on                = register_string("on");
    token_period            = register_string(".");
    token_rate              = register_string("rate_limit");
    token_rbrace            = register_string("}");
    token_require           = register_string("require_match");
    token_requirerdns       = register_string("require_rdns");
    token_semi              = register_string(";");
    token_soft              = register_string("soft");
    token_spamassassin      = register_string("spamassassin");
    token_substitute        = register_string("substitute");
    token_tld               = register_string("tld");
    token_unknown           = register_string("unknown");
    token_uribl             = register_string("uribl");
    token_verify            = register_string("verify");
    token_white             = register_string("white");
    token_white_regex       = register_string("white_regex");
    token_yes               = register_string("yes");
    token_dkim_signer       = register_string("dkim_signer");
    token_dkim_from         = register_string("dkim_from");
    token_signed_white      = register_string("signed_white");
    token_signed_black      = register_string("signed_black");
    token_require_signed    = register_string("require_signed");

    if (gethostname(myhostname, HOST_NAME_MAX+1) != 0) {
        strncpy(myhostname, "localhost", HOST_NAME_MAX+1);
    }
    myhostname[HOST_NAME_MAX] = '\0'; // ensure null termination
    token_myhostname = register_string(myhostname);
}