/*

Copyright (c) 2007 Carl Byington - 510 Software Group, released under
the GPL version 3 or any later version at your choice available at
http://www.gnu.org/licenses/gpl-3.0.txt

*/

#include "includes.h"

#include <arpa/inet.h>
#include <net/if.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/un.h>
#include <unistd.h>
#include <climits>

const char *token_autowhite;
const char *token_black;
const char *token_cctld;
const char *token_content;
const char *token_context;
const char *token_dccbulk;
const char *token_dccfrom;
const char *token_dccgrey;
const char *token_dccto;
const char *token_default;
const char *token_dnsbl;
const char *token_dnsbll;
const char *token_envfrom;
const char *token_envto;
const char *token_filter;
const char *token_generic;
const char *token_host_limit;
const char *token_html_limit;
const char *token_html_tags;
const char *token_ignore;
const char *token_include;
const char *token_inherit;
const char *token_lbrace;
const char *token_mailhost;
const char *token_many;
const char *token_no;
const char *token_off;
const char *token_ok;
const char *token_ok2;
const char *token_on;
const char *token_rate;
const char *token_rbrace;
const char *token_require;
const char *token_semi;
const char *token_soft;
const char *token_spamassassin;
const char *token_substitute;
const char *token_tld;
const char *token_unknown;
const char *token_uribl;
const char *token_verify;
const char *token_white;
const char *token_yes;

const char *token_myhostname;
#ifndef HOST_NAME_MAX
    #define HOST_NAME_MAX 255
#endif
char myhostname[HOST_NAME_MAX+1];

pthread_mutex_t verifier_mutex;     // protect the verifier map
verify_map  verifiers;

pthread_mutex_t whitelister_mutex;  // protect the whitelisters map
whitelister_map whitelisters;

string_set  all_strings;    // owns all the strings, only modified by the config loader thread
const int maxlen = 1000;    // used for snprintf buffers
const int maxsmtp_age =  60;// smtp verify sockets older than this are ancient
const int maxauto_age = 600;// auto whitelister delay before flushing to file
extern int    NULL_SOCKET;
const time_t  ERROR_SMTP_SOCKET_TIME = 600; // number of seconds between attempts to open a socket to an smtp server


int SMTP::writer() {
    #ifdef VERIFY_DEBUG
        log("writer() sees buffer with %s", buffer);
        log("writer() sees error %d", (int)error);
    #endif
    int rs = 0;
    if (!error) {
        int len = strlen(buffer);
        while (rs < len) {
            int ws = write(fd, buffer+rs, len-rs);
            if (ws > 0) {
                rs += ws;
            }
            else {
                // peer closed the socket!
                rs = 0;
                error = true;
                break;
            }
        }
    }
    return rs;
}


int SMTP::reader() {
    // read some bytes terminated by lf or end of buffer.
    // we may have a multi line response or part thereof in the buffer.
    #ifdef VERIFY_DEBUG
        log("reader() sees error %d", (int)error);
    #endif
    if (error) return 0;
    int len = maxlen-1; // room for null terminator
    while (pending < len) {
        int ws = read(fd, buffer+pending, len-pending);
        if (ws > 0) {
            pending += ws;
            if (buffer[pending-1] == '\n') break;
        }
        else {
            // peer closed the socket!
            pending = 0;
            error = true;
            break;
        }
    }
    buffer[pending] = '\0';
    #ifdef VERIFY_DEBUG
        log("reader() sees buffer with %s", buffer);
    #endif
    return pending;
}


int SMTP::read_line() {
    char *lf = strchr(buffer, '\n');
    if (!lf) {
        reader();  // get a lf
        lf = strchr(buffer, '\n');
        if (!lf) lf = buffer + pending - 1;
    }
    return (lf-buffer)+1;   // number of bytes in this line
}


void SMTP::flush_line(int r) {
    if (pending > r) memmove(buffer, buffer+r, pending-r);
    pending -= r;
}


int SMTP::read_response() {
    pending = 0;
    buffer[pending] = '\0';
    while (true) {
        int r = read_line();
        #ifdef VERIFY_DEBUG
            log("read_response() sees line with %s", buffer);
            log("read_response() sees line length %d", r);
        #endif
        if (r == 0) return 0;   // failed to read any bytes
        if ((r > 4) && (buffer[3] == '-')) {
            flush_line(r);
            continue;
        }
        return atoi(buffer);
    }
    return 0;
}


int SMTP::cmd(const char *c) {
    if (c) {
        init();
        append(c);
    }
    append("\r\n");
    writer();
    return read_response();
}


int SMTP::helo() {
    if (read_response() != 220) return 0;
    init();
    append("HELO ");
    append(token_myhostname);
    return cmd(NULL);
}


int SMTP::rset() {
    int rc = cmd("RSET");
    efrom[0] = '\0';
    return rc;
}


int SMTP::from(const char *f) {
    // the mail from address was originally passed in from sendmail enclosed in
    // <>. to_lower_string() removed the <> and converted the rest to lowercase,
    // except in the case of an empty return path, which was left as the two
    // character string <>.
    if (strncmp(efrom, f, maxlen)) {
        rset();
        strncpy(efrom, f, maxlen);
        init();
        append("MAIL FROM:<");
        if (*f != '<') append(f);
        append(">");
        return cmd(NULL);
    }
    return 250; // pretend it worked
}


int SMTP::rcpt(const char *t) {
    init();
    append("RCPT TO:<");
    append(t);
    append(">");
    return cmd(NULL);
}


int SMTP::quit() {
    return cmd("QUIT");
}


void SMTP::closefd() {
    shutdown(fd, SHUT_RDWR);
    close(fd);
}


#ifdef VERIFY_DEBUG
    void SMTP::log(const char *m, int v) {
        char buf[maxlen];
        snprintf(buf, maxlen, m, v);
        my_syslog(buf);
    }


    void SMTP::log(const char *m, const char *v) {
        char buf[maxlen];
        snprintf(buf, maxlen, m, v);
        my_syslog(buf);
    }
#endif


////////////////////////////////////////////////
// smtp verifier so backup mx machines can see the valid users
//
VERIFY::VERIFY(const char *h) {
    host     = h;
    last_err = 0;
    pthread_mutex_init(&mutex, 0);
}


void VERIFY::closer() {
    bool ok = true;
    while (ok) {
        SMTP *conn = NULL;
        pthread_mutex_lock(&mutex);
            if (connections.empty()) {
                ok = false;
            }
            else {
                conn = connections.front();
                time_t now = time(NULL);
                if ((now - conn->get_stamp()) > maxsmtp_age) {
                    // this connection is ancient, remove it
                    connections.pop_front();
                }
                else {
                    ok   = false;
                    conn = NULL;
                }
            }
        pthread_mutex_unlock(&mutex);
        // avoid doing this work inside the mutex lock
        if (conn) {
            #ifdef VERIFY_DEBUG
                conn->log("closer() closes ancient %d", conn->get_fd());
            #endif
            delete conn;
        }
    }
}


SMTP* VERIFY::get_connection() {
    SMTP *conn = NULL;
    pthread_mutex_lock(&mutex);
        if (!connections.empty()) {
            conn = connections.front();
            connections.pop_front();
            #ifdef VERIFY_DEBUG
                conn->log("get_connection() %d from cache", conn->get_fd());
            #endif
        }
    pthread_mutex_unlock(&mutex);
    if (conn) return conn;
    int sock = NULL_SOCKET;
    if ((time(NULL) - last_err) > ERROR_SMTP_SOCKET_TIME) {
        // nothing recent, maybe this time it will work
        hostent *h = gethostbyname(host);
        if (h) {
            sockaddr_in server;
            server.sin_family = h->h_addrtype;
            server.sin_port   = htons(25);
            memcpy(&server.sin_addr, h->h_addr_list[0], h->h_length);
            sock = socket(PF_INET, SOCK_STREAM, 0);
            if (sock != NULL_SOCKET) {
                bool rc = (connect(sock, (sockaddr *)&server, sizeof(server)) == 0);
                if (!rc) {
                    shutdown(sock, SHUT_RDWR);
                    close(sock);
                    sock = NULL_SOCKET;
                    last_err = time(NULL);
                }
            }
            else last_err = time(NULL);
        }
        else last_err = time(NULL);
    }
    if (sock != NULL_SOCKET) {
        conn = new SMTP(sock);
        #ifdef VERIFY_DEBUG
            conn->log("get_connection() %d new socket", conn->get_fd());
        #endif
        if (conn->helo() == 250) return conn;
        delete conn;
    }
    return NULL;
}


void VERIFY::put_connection(SMTP *conn) {
    if (conn->err()) {
        #ifdef VERIFY_DEBUG
            conn->log("put_socket() %d with error, close it", conn->get_fd());
        #endif
        delete conn;
        last_err = time(NULL);
    }
    else {
        #ifdef VERIFY_DEBUG
            conn->log("put_socket() %d", conn->get_fd());
        #endif
        conn->now();
        pthread_mutex_lock(&mutex);
            connections.push_back(conn);
        pthread_mutex_unlock(&mutex);
    }
}


bool VERIFY::ok(const char *from, const char *to) {
    if (host == token_myhostname) return true;
    SMTP *conn = get_connection();
    if (!conn) return true;    // cannot verify right now, we have socket errors
    int rc;
    rc = conn->from(from);
    #ifdef VERIFY_DEBUG
        conn->log("verify::ok() from sees %d", rc);
    #endif
    if (rc != 250) {
        conn->rset();
        put_connection(conn);
        return (rc >= 500) ? false : true;
    }
    rc = conn->rcpt(to);
    #ifdef VERIFY_DEBUG
        conn->log("verify::ok() rcpt sees %d", rc);
    #endif
    put_connection(conn);
    return (rc >= 500) ? false : true;
}


////////////////////////////////////////////////
// setup a new smtp verify host
//
VERIFYP add_verify_host(const char *host);
VERIFYP add_verify_host(const char *host) {
    VERIFYP rc = NULL;
    pthread_mutex_lock(&verifier_mutex);
        verify_map::iterator i = verifiers.find(host);
        if (i == verifiers.end()) {
            rc = new VERIFY(host);
            verifiers[host] = rc;
        }
        else rc = (*i).second;
    pthread_mutex_unlock(&verifier_mutex);
    return rc;
}


////////////////////////////////////////////////
// thread to check for verify hosts with old sockets that we can close
//
void* verify_closer(void *arg) {
    while (true) {
        sleep(maxsmtp_age);
        pthread_mutex_lock(&verifier_mutex);
            for (verify_map::iterator i=verifiers.begin(); i!=verifiers.end(); i++) {
                VERIFYP v = (*i).second;
                v->closer();
            }
        pthread_mutex_unlock(&verifier_mutex);
    }
    return NULL;
}


////////////////////////////////////////////////
// automatic whitelister
//
WHITELISTER::WHITELISTER(const char *f, int d) {
    fn       = f;
    days     = d;
    pthread_mutex_init(&mutex, 0);
    need     = false;
    loaded   = time(NULL);
    merge();
}


void WHITELISTER::merge() {
    time_t now = time(NULL);
    ifstream ifs;
    ifs.open(fn);
    if (!ifs.fail()) {
        const int maxlen = 1000;
        char buf[maxlen];
        while (ifs.getline(buf, maxlen)) {
            char *p = strchr(buf, ' ');
            if (p) {
                *p = '\0';
                char   *who = strdup(buf);
                time_t when = atoi(p+1);
                if ((when == 0) || (when > now)) when = now;
                autowhite_sent::iterator i = rcpts.find(who);
                if (i == rcpts.end()) {
                    rcpts[who] = when;
                }
                else {
                    time_t wh = (*i).second;
                    if ((when == 1) || (when > wh)) (*i).second = when;
                    free(who);
                }
            }
        }
    }
    ifs.close();
}


void WHITELISTER::writer() {
    pthread_mutex_lock(&mutex);
        time_t limit = time(NULL) - days*86400;

        // check for manually modified autowhitelist file
        struct stat st;
        if (stat(fn, &st)) need = true; // file has disappeared
        else if (st.st_mtime > loaded) {
            // file has been manually updated, merge new entries
            merge();
            need = true;
        }

        // purge old entries
        for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end();) {
            time_t when = (*i).second;
            if (when < limit) {
                const char *who = (*i).first;
                free((void*)who);
                rcpts.erase(i++);
                need = true;
            }
            else i++;
        }

        if (need) {
            // dump the file
            ofstream ofs;
            ofs.open(fn);
            if (!ofs.fail()) {
                for (autowhite_sent::iterator i=rcpts.begin(); i!=rcpts.end(); i++) {
                    const char *who = (*i).first;
                    int  when = (*i).second;
                    if (!strchr(who, ' ')) {
                        ofs << who << " " << when << endl;
                    }
                }
            }
            ofs.close();
            need = false;
            loaded = time(NULL);    // update load time
        }
    pthread_mutex_unlock(&mutex);
}


void WHITELISTER::sent(const char *to) {
    // we take ownership of the string
    pthread_mutex_lock(&mutex);
        need = true;
        autowhite_sent::iterator i = rcpts.find(to);
        if (i == rcpts.end()) {
            rcpts[to] = time(NULL);
        }
        else {
            (*i).second = time(NULL);
            free((void*)to);
        }
    pthread_mutex_unlock(&mutex);
}


bool WHITELISTER::is_white(const char *from) {
    pthread_mutex_lock(&mutex);
        autowhite_sent::iterator i = rcpts.find(from);
        bool rc = (i != rcpts.end());
    pthread_mutex_unlock(&mutex);
    return rc;
}


////////////////////////////////////////////////
// setup a new auto whitelister file
//
WHITELISTERP add_whitelister_file(const char *fn, int days);
WHITELISTERP add_whitelister_file(const char *fn, int days) {
    WHITELISTERP rc = NULL;
    pthread_mutex_lock(&whitelister_mutex);
        whitelister_map::iterator i = whitelisters.find(fn);
        if (i == whitelisters.end()) {
            rc = new WHITELISTER(fn, days);
            whitelisters[fn] = rc;
        }
        else {
            rc = (*i).second;
            rc->set_days(days);
        }
    pthread_mutex_unlock(&whitelister_mutex);
    return rc;
}


////////////////////////////////////////////////
// thread to check for whitelister hosts with old sockets that we can close
//
void* whitelister_writer(void *arg) {
    while (true) {
        sleep(maxauto_age);
        pthread_mutex_lock(&whitelister_mutex);
            for (whitelister_map::iterator i=whitelisters.begin(); i!=whitelisters.end(); i++) {
                WHITELISTERP v = (*i).second;
                v->writer();
            }
        pthread_mutex_unlock(&whitelister_mutex);
    }
    return NULL;
}


DELAYWHITE::DELAYWHITE(const char *loto_, WHITELISTERP w_, CONTEXTP con_) {
    loto = loto_;
    w    = w_;
    con  = con_;
}


DNSBL::DNSBL(const char *n, const char *s, const char *m) {
    name    = n;
    suffix  = s;
    message = m;
}


bool DNSBL::operator==(const DNSBL &rhs) {
    return (strcmp(name,    rhs.name)    == 0) &&
           (strcmp(suffix,  rhs.suffix)  == 0) &&
           (strcmp(message, rhs.message) == 0);
}


CONFIG::CONFIG() {
    reference_count    = 0;
    generation         = 0;
    load_time          = 0;
    default_context    = NULL;
}


CONFIG::~CONFIG() {
    if (debug_syslog) {
        char buf[maxlen];
        snprintf(buf, sizeof(buf), "freeing memory for old configuration generation %d", generation);
        my_syslog(buf);
    }
    for (context_list::iterator i=contexts.begin(); i!=contexts.end(); i++) {
        CONTEXT *c = *i;
        delete c;
    }
}


void CONFIG::add_context(CONTEXTP con) {
    contexts.push_back(con);
    if (!default_context && !con->get_parent()) {
        // first global context
        default_context = con;
    }
}


void CONFIG::add_to(const char *to, CONTEXTP con) {
    context_map::iterator i = env_to.find(to);
    if (i != env_to.end()) {
        CONTEXTP  c = (*i).second;
        if ((c != con) && (c != con->get_parent())) {
            if (debug_syslog) {
                char oldname[maxlen];
                char newname[maxlen];
                const char *oldn = c->get_full_name(oldname, maxlen);
                const char *newn = con->get_full_name(newname, maxlen);
                char buf[maxlen*3];
                snprintf(buf, maxlen*3, "both %s and %s claim envelope to %s, the second one wins", oldn, newn, to);
                my_syslog(buf);
            }
        }
    }
    env_to[to] = con;
}


CONTEXTP CONFIG::find_context(const char *to) {
    context_map::iterator i = env_to.find(to);
    if (i != env_to.end()) return (*i).second;      // found user@domain key
    const char *x = strchr(to, '@');
    if (x) {
        x++;
        i = env_to.find(x);
        if (i != env_to.end()) return (*i).second;  // found domain key
        size_t len = x - to;
        char user[len+1];
        memcpy(user, to, len);
        user[len] = '\0';
        i = env_to.find(user);
        if (i != env_to.end()) return (*i).second;  // found user@ key
    }
    return default_context;
}


void CONFIG::dump() {
    bool spamass = false;
    if (default_context) default_context->dump(true, spamass);
    for (context_list::iterator i=contexts.begin(); i!=contexts.end(); i++) {
        CONTEXTP c = *i;
        CONTEXTP p = c->get_parent();
        if (!p && (c != default_context)) c->dump(false, spamass);
    }
    char buf[maxlen];
    for (context_map::iterator i=env_to.begin(); i!=env_to.end(); i++) {
        const char *to = (*i).first;
        CONTEXTP   con = (*i).second;
        printf("// envelope to %s \t-> context %s \n", to, con->get_full_name(buf,maxlen));
    }
    if (spamass && (spamc == spamc_empty)) {
        printf("// *** warning - spamassassin filtering requested, but spamc not found by autoconf.\n");
    }
}


CONTEXT::CONTEXT(CONTEXTP parent_, const char *name_) {
    parent              = parent_;
    name                = name_;
    verify_host         = NULL;
    verifier            = NULL;
    generic_regx        = NULL;
    generic_message     = NULL;
    autowhite_file      = NULL;
    whitelister         = NULL;
    env_from_default    = (parent) ? token_inherit : token_unknown;
    content_filtering   = (parent) ? parent->content_filtering : false;
    content_suffix      = NULL;
    content_message     = NULL;
    uribl_suffix        = NULL;
    uribl_message       = NULL;
    host_limit          = (parent) ? parent->host_limit  : 0;
    host_limit_message  = NULL;
    host_random         = (parent) ? parent->host_random : false;
    tag_limit           = (parent) ? parent->tag_limit   : 0;
    tag_limit_message   = NULL;
    spamassassin_limit  = (parent) ? parent->spamassassin_limit : 0;
    require_match       = (parent) ? parent->require_match      : false;
    dcc_greylist        = (parent) ? parent->dcc_greylist       : false;
    dcc_bulk_threshold  = (parent) ? parent->dcc_bulk_threshold : 0;
    default_rcpt_rate   = INT_MAX;
}


CONTEXT::~CONTEXT() {
    for (dnsblp_map::iterator i=dnsbl_names.begin(); i!=dnsbl_names.end(); i++) {
        DNSBLP d = (*i).second;
        // delete the underlying DNSBL objects.
        delete d;
    }
    if (generic_regx) regfree(&generic_pattern);
}


bool CONTEXT::is_parent(CONTEXTP p) {
    if (p == parent) return true;
    if (!parent) return false;
    return parent->is_parent(p);
}


const char *CONTEXT::get_full_name(char *buffer, int size) {
    if (!parent) return name;
    char buf[maxlen];
    snprintf(buffer, size, "%s.%s", parent->get_full_name(buf, maxlen), name);
    return buffer;
}


bool CONTEXT::set_generic(const char *regx, const char *msg)
{
    int rc = 0;
    if (generic_regx) regfree(&generic_pattern);
    generic_regx    = regx;
    generic_message = msg;
    if (generic_regx) {
        rc = regcomp(&generic_pattern, regx, REG_NOSUB | REG_ICASE | REG_EXTENDED);
    }
    return rc;  // true iff bad pattern
}


const char *CONTEXT::generic_match(const char *client)
{
    if (!client) return NULL;   // allow missing _ macro, which will disable generic checking
    if (parent && !generic_regx) return parent->generic_match(client);
    if (!generic_regx)           return NULL;
    if (0 == regexec(&generic_pattern, client, 0, NULL, 0)) {
        return generic_message;
    }
    return NULL;
}


bool CONTEXT::cover_env_to(const char *to) {
    const char *x = strchr(to, '@');
    if (x) x++;
    else   x = to;
    if (*x == '\0') return true;                // always allow covering addresses with no domain name, eg abuse@
    if (!parent && env_to.empty()) return true; // empty env_to at global level covers everything
    string_set::iterator i = env_to.find(x);
    if (i != env_to.end()) return true;         // we cover the entire domain
    if (x != to) {
        i = env_to.find(to);
        if (i != env_to.end()) return true;     // we cover the specific email address
    }
    return false;
}


VERIFYP CONTEXT::find_verify(const char *to) {
    if (verifier && (verify_host != token_myhostname) && cover_env_to(to))
        return verifier;
    else if (parent)
        return parent->find_verify(to);
    else
        return NULL;
}


WHITELISTERP CONTEXT::find_autowhite(const char *from, const char *to) {
    if (whitelister && cover_env_to(to) && !cover_env_to(from))
        return whitelister;
    else if (parent)
        return parent->find_autowhite(from, to);
    else
        return NULL;
}


int CONTEXT::find_rate(const char *user) {
    if (rcpt_per_hour.empty()) return default_rcpt_rate;
    rcpt_rates::iterator i = rcpt_per_hour.find(user);
    return (i == rcpt_per_hour.end()) ? default_rcpt_rate : (*i).second;
}


const char *CONTEXT::find_from(const char *from, bool update_white) {
    WHITELISTERP w = whitelister;
    CONTEXTP     p = parent;
    while (!w && p) {
        w = p->whitelister;
        p = p->parent;
    }
    if (w && w->is_white(from)) {
        if (update_white) {
            // update senders timestamp to extend the whitelisting period
            if (debug_syslog > 1) {
                char buf[maxlen];
                char msg[maxlen];
                snprintf(msg, sizeof(msg), "extend whitelist reply from <%s> in context %s", from, get_full_name(buf,maxlen));
                my_syslog(msg);
            }
            w->sent(strdup(from));
        }
        return token_white;
    }
    const char *rc = env_from_default;
    string_map::iterator i = env_from.find(from);
    if (i != env_from.end()) rc = (*i).second;  // found user@domain key
    else {
        char *x = strchr(from, '@');
        if (x) {
            x++;
            i = env_from.find(x);
            if (i != env_from.end()) rc = (*i).second;  // found domain key
            else {
                char y = *x;
                *x = '\0';
                i = env_from.find(from);
                *x = y;
                if (i != env_from.end()) rc = (*i).second;  // found user@ key
            }
        }
    }
    if ((rc == token_inherit) && parent) return parent->find_from(from);
    return (rc == token_inherit) ? token_unknown : rc;
}


CONTEXTP CONTEXT::find_context(const char *from) {
    context_map::iterator i = env_from_context.find(from);
    if (i != env_from_context.end()) return (*i).second;        // found user@domain key
    char *x = strchr(from, '@');
    if (x) {
        x++;
        i = env_from_context.find(x);
        if (i != env_from_context.end()) return (*i).second;    // found domain key
        char y = *x;
        *x = '\0';
        i = env_from_context.find(from);
        *x = y;
        if (i != env_from_context.end()) return (*i).second;    // found user@ key
    }
    return this;
}


CONTEXTP CONTEXT::find_from_context_name(const char *name) {
    context_map::iterator i = children.find(name);
    if (i != children.end()) return (*i).second;
    return NULL;
}


DNSBLP CONTEXT::find_dnsbl(const char *name) {
    dnsblp_map::iterator i = dnsbl_names.find(name);
    if (i != dnsbl_names.end()) return (*i).second;
    if (parent) return parent->find_dnsbl(name);
    return NULL;
}


const char* CONTEXT::get_content_suffix() {
    if (!content_suffix && parent) return parent->get_content_suffix();
    return content_suffix;
}


const char* CONTEXT::get_uribl_suffix() {
    if (!uribl_suffix && parent) return parent->get_uribl_suffix();
    return uribl_suffix;
}


const char* CONTEXT::get_content_message() {
    if (!content_message && parent) return parent->get_content_message();
    return content_message;
}


const char* CONTEXT::get_uribl_message() {
    if (!uribl_message && parent) return parent->get_uribl_message();
    return uribl_message;
}


string_set& CONTEXT::get_content_host_ignore() {
    if (content_host_ignore.empty() && parent) return parent->get_content_host_ignore();
    return content_host_ignore;
}


string_set& CONTEXT::get_content_cctlds() {
    if (content_cctlds.empty() && parent) return parent->get_content_cctlds();
    return content_cctlds;
}

string_set& CONTEXT::get_content_tlds() {
    if (content_tlds.empty() && parent) return parent->get_content_tlds();
    return content_tlds;
}


string_set& CONTEXT::get_html_tags() {
    if (html_tags.empty() && parent) return parent->get_html_tags();
    return html_tags;
}


dnsblp_list& CONTEXT::get_dnsbl_list() {
    if (dnsbl_list.empty() && parent) return parent->get_dnsbl_list();
    return dnsbl_list;
}


bool CONTEXT::acceptable_content(recorder &memory, int score, int bulk, string& msg) {
    if (spamassassin_limit && (score > spamassassin_limit)) {
        char buf[maxlen];
        snprintf(buf, sizeof(buf), "Mail rejected - spam assassin score %d", score);
        msg = string(buf);
        return false;
    }
    if (dcc_bulk_threshold && (bulk >= dcc_bulk_threshold)) {
        char buf[maxlen];
        snprintf(buf, sizeof(buf), "Mail rejected - dcc score %d", bulk);
        msg = string(buf);
        return false;
    }
    if (memory.excessive_bad_tags(tag_limit)) {
        msg = string(tag_limit_message);
        return false;
    }
    if (!host_random && memory.excessive_hosts(host_limit)) {
        msg = string(host_limit_message);
        return false;
    }
    return true;
}


void CONTEXT::dump(bool isdefault, bool &spamass, int level) {
    char indent[maxlen];
    int i = min(maxlen-1, level*4);
    memset(indent, ' ', i);
    indent[i] = '\0';
    char buf[maxlen];
    const char *fullname = get_full_name(buf,maxlen);
    printf("%s context %s { \t// %s\n", indent, name, fullname);

    for (dnsblp_map::iterator i=dnsbl_names.begin(); i!=dnsbl_names.end(); i++) {
        const char *n = (*i).first;
        DNSBL &d = *(*i).second;
        printf("%s     dnsbl %s %s \"%s\"; \n", indent, n, d.suffix, d.message);
    }

    dnsblp_list dl = get_dnsbl_list();
    if (!dl.empty()) {
        printf("%s     dnsbl_list", indent);
        for (dnsblp_list::iterator i=dl.begin(); i!=dl.end(); i++) {
            DNSBL &d = *(*i);
            printf(" %s", d.name);
        }
        printf("; \n");
    }

    if (content_filtering) {
        printf("%s     content on { \n", indent);
        if (content_suffix) {
            printf("%s         filter %s \"%s\"; \n", indent, content_suffix, content_message);
        }
        if (uribl_suffix) {
            printf("%s         uribl %s \"%s\"; \n", indent, uribl_suffix, uribl_message);
        }
        if (!content_host_ignore.empty()) {
            printf("%s         ignore { \n", indent);
            for (string_set::iterator i=content_host_ignore.begin(); i!=content_host_ignore.end(); i++) {
                printf("%s             %s; \n", indent, *i);
            }
            printf("%s         }; \n", indent);
        }
        if (!content_cctlds.empty()) {
            printf("%s         cctld { \n", indent);
            printf("%s             ", indent);
            for (string_set::iterator i=content_cctlds.begin(); i!=content_cctlds.end(); i++) {
                printf("%s; ", *i);
            }
            printf("\n%s         }; \n", indent);
        }
        if (!content_tlds.empty()) {
            printf("%s         tld { \n", indent);
            printf("%s             ", indent);
            for (string_set::iterator i=content_tlds.begin(); i!=content_tlds.end(); i++) {
                printf("%s; ", *i);
            }
            printf("\n%s         }; \n", indent);
        }
        if (!html_tags.empty()) {
            printf("%s         html_tags { \n", indent);
            printf("%s             ", indent);
            for (string_set::iterator i=html_tags.begin(); i!=html_tags.end(); i++) {
                printf("%s; ", *i);
            }
            printf("\n%s         }; \n", indent);
        }
        if (host_limit_message) {
            printf("%s         host_limit on %d \"%s\"; \n", indent, host_limit, host_limit_message);
        }
        else if (host_random) {
            printf("%s         host_limit soft %d; \n", indent, host_limit);
        }
        else {
            printf("%s         host_limit off; \n", indent);
        }
        if (tag_limit_message) {
            printf("%s         html_limit on %d \"%s\"; \n", indent, tag_limit, tag_limit_message);
        }
        else {
            printf("%s         html_limit off; \n", indent);
        }
        printf("%s         spamassassin %d; \n", indent, spamassassin_limit);
        printf("%s         require_match %s; \n", indent, (require_match) ? "yes" : "no");
        printf("%s         dcc_greylist  %s; \n", indent, (dcc_greylist)  ? "yes" : "no");
        if (dcc_bulk_threshold == 0)            printf("%s         dcc_bulk_threshold off; \n", indent);
        else if (dcc_bulk_threshold >= dccbulk) printf("%s         dcc_bulk_threshold many; \n", indent);
        else                                    printf("%s         dcc_bulk_threshold %d; \n", indent, dcc_bulk_threshold);
        printf("%s     }; \n", indent);
        spamass |= (spamassassin_limit != 0);
        }
    else {
        printf("%s     content off {}; \n", indent);
    }

    printf("%s     env_to { \t// %s\n", indent, fullname);
    for (string_set::iterator i=env_to.begin(); i!=env_to.end(); i++) {
        printf("%s         %s; \n", indent, *i);
    }
    printf("%s     }; \n", indent);

    if (verify_host) {
        printf("%s     verify %s; \n", indent, verify_host);
    }

    if (generic_regx) {
        printf("%s     generic \"%s\"  \n", indent, generic_regx);
        printf("%s             \"%s\"; \n", indent, generic_message);
    }

    if (autowhite_file && whitelister) {
        printf("%s     autowhite %d %s; \n", indent, whitelister->get_days(), autowhite_file);
    }

    for (context_map::iterator i=children.begin(); i!=children.end(); i++) {
        CONTEXTP c = (*i).second;
        c->dump(false, spamass, level+1);
    }

    printf("%s     env_from %s { \t// %s\n", indent, env_from_default, fullname);
    if (!env_from.empty()) {
        printf("%s     // white/black/unknown \n", indent);
        for (string_map::iterator i=env_from.begin(); i!=env_from.end(); i++) {
            const char *f = (*i).first;
            const char *t = (*i).second;
            printf("%s         %s \t%s; \n", indent, f, t);
        }
    }
    if (!env_from_context.empty()) {
        printf("%s     // child contexts \n", indent);
        for (context_map::iterator j=env_from_context.begin(); j!=env_from_context.end(); j++) {
            const char    *f = (*j).first;
            CONTEXTP t = (*j).second;
            printf("%s         %s \t%s; \n", indent, f, t->name);
        }
    }
    printf("%s     }; \n", indent);

    if (isdefault) {
        printf("%s     rate_limit %d { \n", indent, default_rcpt_rate);
        for (rcpt_rates::iterator j=rcpt_per_hour.begin(); j!=rcpt_per_hour.end(); j++) {
            const char *u = (*j).first;
            int         l = (*j).second;
            printf("%s         \"%s\" \t%d; \n", indent, u, l);
        }
        printf("%s     }; \n", indent);
    }

    printf("%s }; \n", indent);
}


////////////////////////////////////////////////
// helper to discard the strings held by a string_set
//
void discard(string_set &s) {
    for (string_set::iterator i=s.begin(); i!=s.end(); i++) {
        free((void*)*i);
    }
    s.clear();
}


////////////////////////////////////////////////
// helper to register a string in a string set
//
const char* register_string(string_set &s, const char *name) {
    string_set::iterator i = s.find(name);
    if (i != s.end()) return *i;
    char *x = strdup(name);
    s.insert(x);
    return x;
}


////////////////////////////////////////////////
// register a global string
//
const char* register_string(const char *name) {
    return register_string(all_strings, name);
}


////////////////////////////////////////////////
// clear all global strings, helper for valgrind checking
//
void clear_strings() {
    discard(all_strings);
}


////////////////////////////////////////////////
//
bool tsa(TOKEN &tok, const char *token);
bool tsa(TOKEN &tok, const char *token) {
    const char *have = tok.next();
    if (have == token) return true;
    tok.token_error(token, have);
    return false;
}


////////////////////////////////////////////////
//
bool parse_dnsbl(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnsbl(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *name = tok.next();
    const char *suf  = tok.next();
    const char *msg  = tok.next();
    if (!tsa(tok, token_semi)) return false;
    DNSBLP dnsnew = new DNSBL(name, suf, msg);
    DNSBLP dnsold = me.find_dnsbl(name);
    if (dnsold && (*dnsold == *dnsnew)) {
        // duplicate redefinition, ignore it
        delete dnsnew;
        return true;
    }
    me.add_dnsbl(name, dnsnew);
    return true;
}


////////////////////////////////////////////////
//
bool parse_dnsbll(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_dnsbll(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_semi) break;
        DNSBLP dns = me.find_dnsbl(have);
        if (dns) {
            me.add_dnsbl(dns);
        }
        else {
            tok.token_error("dnsbl name", have);
            return false;
        }
    }
    return true;
}


////////////////////////////////////////////////
//
bool parse_content(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_content(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *setting = tok.next();
    if (setting == token_on) {
        me.set_content_filtering(true);
    }
    else if (setting == token_off) {
        me.set_content_filtering(false);
    }
    else {
        tok.token_error("on/off", setting);
        return false;
    }
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_filter) {
            const char *suffix = tok.next();
            const char *messag = tok.next();
            me.set_content_suffix(suffix);
            me.set_content_message(messag);
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_uribl) {
            const char *suffix = tok.next();
            const char *messag = tok.next();
            me.set_uribl_suffix(suffix);
            me.set_uribl_message(messag);
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_ignore) {
            if (!tsa(tok, token_lbrace)) return false;
            while (true) {
                if (!have) break;
                const char *have = tok.next();
                if (have == token_rbrace) break;  // done
                me.add_ignore(have);
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_tld) {
            if (!tsa(tok, token_lbrace)) return false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) break;  // done
                me.add_tld(have);
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_cctld) {
            if (!tsa(tok, token_lbrace)) return false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) break;  // done
                me.add_cctld(have);
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_html_tags) {
            if (!tsa(tok, token_lbrace)) return false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) {
                    break;  // done
                }
                else {
                    me.add_tag(have);                           // base version
                    char buf[200];
                    snprintf(buf, sizeof(buf), "/%s", have);
                    me.add_tag(register_string(buf));           // leading /
                    snprintf(buf, sizeof(buf), "%s/", have);
                    me.add_tag(register_string(buf));           // trailing /
                }
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_html_limit) {
            have = tok.next();
            if (have == token_on) {
                me.set_tag_limit(tok.nextint());
                me.set_tag_message(tok.next());
            }
            else if (have == token_off) {
                me.set_tag_limit(0);
                me.set_tag_message(NULL);
            }
            else {
                tok.token_error("on/off", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_host_limit) {
            have = tok.next();
            if (have == token_on) {
                me.set_host_limit(tok.nextint());
                me.set_host_message(tok.next());
                me.set_host_random(false);
            }
            else if (have == token_off) {
                me.set_host_limit(0);
                me.set_host_message(NULL);
                me.set_host_random(false);
            }
            else if (have == token_soft) {
                me.set_host_limit(tok.nextint());
                me.set_host_message(NULL);
                me.set_host_random(true);
            }
            else {
                tok.token_error("on/off/soft", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_spamassassin) {
            me.set_spamassassin_limit(tok.nextint());
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_require) {
            have = tok.next();
                 if (have == token_yes) me.set_require(true);
            else if (have == token_no)  me.set_require(false);
            else {
                tok.token_error("yes/no", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_dccgrey) {
            have = tok.next();
                 if (have == token_yes) me.set_grey(true);
            else if (have == token_no)  me.set_grey(false);
            else {
                tok.token_error("yes/no", have);
                return false;
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_dccbulk) {
            have = tok.next();
                 if (have == token_off) me.set_bulk(0);
            else if (have == token_many) me.set_bulk(dccbulk);
            else {
                char *e;
                long i = strtol(have, &e, 10);
                if (*e != '\0') {
                    tok.token_error("integer", have);
                    return false;
                }
                me.set_bulk((int)i);
            }
            if (!tsa(tok, token_semi)) return false;
        }
        else if (have == token_rbrace) {
            break;  // done
        }
        else {
            tok.token_error("content keyword", have);
            return false;
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_envto(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_envto(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        if (have == token_semi) {
            // optional separators
        }
        else if (have == token_dccto) {
            const char *flavor = tok.next();
            if (!tsa(tok, token_lbrace)) return false;
            bool keeping = false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) break;
                if (have == flavor) {
                    keeping = true;
                    continue;
                }
                else if ((have == token_ok) || (have == token_ok2) || (have == token_many)) {
                    keeping = false;
                    continue;
                }
                if (have == token_envto) {
                    have = tok.next();
                    if (keeping) {
                        if (me.allow_env_to(have)) {
                            me.add_to(have);
                            dc.add_to(have, &me);
                        }
                    }
                }
              //else if (have == token_substitute) {
              //    if (tok.next() == token_mailhost) {
              //        have = tok.next();
              //        if (keeping) {
              //            if (me.allow_env_to(have)) {
              //                me.add_to(have);
              //                dc.add_to(have, &me);
              //            }
              //        }
              //    }
              //}
                tok.skipeol();
            }
        }
        else if (me.allow_env_to(have)) {
            me.add_to(have);
            dc.add_to(have, &me);
        }
        else {
            tok.token_error("user@ or user@domain.tld or domain.tld where domain.tld allowed by parent context", have);
            return false;
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_verify(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_verify(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *host = tok.next();
    if (!tsa(tok, token_semi)) return false;
    me.set_verify(host);
    me.set_verifier(add_verify_host(host));
    return true;
}


////////////////////////////////////////////////
//
bool parse_generic(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_generic(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *regx = tok.next();
    const char *msg  = tok.next();
    if (!tsa(tok, token_semi)) return false;
    if (me.set_generic(regx, msg)) {
        tok.token_error("invalid regular expression %s", regx, regx);
        return false;
    }
    return true;
}


////////////////////////////////////////////////
//
bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_autowhite(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    int days = tok.nextint();
    const char *fn = tok.next();
    if (!tsa(tok, token_semi)) return false;
    me.set_autowhite(fn);
    me.set_whitelister(add_whitelister_file(fn, days));
    return true;
}


////////////////////////////////////////////////
//
bool parse_envfrom(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_envfrom(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *st = tok.next();
    if ((st == token_black) || (st == token_white) || (st == token_unknown) || (st == token_inherit)) {
        me.set_from_default(st);
    }
    else {
        tok.push(st);
    }
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        if (have == token_semi) {
            // optional separators
        }
        else if (have == token_dccfrom) {
            if (!tsa(tok, token_lbrace)) return false;
            bool keeping = false;
            bool many = false;
            while (true) {
                const char *have = tok.next();
                if (!have) break;
                if (have == token_rbrace) break;
                if (have == token_ok) {
                    keeping = true;
                    many    = false;
                    continue;
                }
                else if (have == token_many) {
                    keeping = true;
                    many    = true;
                    continue;
                }
                else if (have == token_ok2) {
                    keeping = false;
                    continue;
                }
                if (have == token_envfrom) {
                    have = tok.next();
                    if (keeping) {
                        me.add_from(have, (many) ? token_black : token_white);
                    }
                }
                else if (have == token_substitute) {
                    if (tok.next() == token_mailhost) {
                        have = tok.next();
                        me.add_from(have, (many) ? token_black : token_white);
                    }
                }
                tok.skipeol();
            }
        }
        else {
            // may be a valid email address or domain name
            const char *st = tok.next();
            if ((st == token_white) || (st == token_black) || (st == token_unknown) || (st == token_inherit)) {
                me.add_from(have, st);
            }
            else {
                CONTEXTP con = me.find_from_context_name(st);
                if (con) {
                    me.add_from_context(have, con);
                }
                else {
                    tok.token_error("white/black/unknown/inherit or child context name", st);
                    return false;
                }
            }
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me);
bool parse_rate(TOKEN &tok, CONFIG &dc, CONTEXT &me) {
    const char *def = tok.next();
    tok.push(def);
    if (def != token_lbrace) me.set_default_rate(tok.nextint());
    if (!tsa(tok, token_lbrace)) return false;
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;
        if (have == token_semi) {
            // optional separators
        }
        else {
            me.add_rate(have, tok.nextint());
        }
    }
    return tsa(tok, token_semi);
}


////////////////////////////////////////////////
//
bool parse_context(TOKEN &tok, CONFIG &dc, CONTEXTP parent);
bool parse_context(TOKEN &tok, CONFIG &dc, CONTEXTP parent) {
    const char *name = tok.next();
    if (!tsa(tok, token_lbrace)) return false;
    CONTEXTP con = new CONTEXT(parent, name);

    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_rbrace) break;  // done
        if (have == token_dnsbl) {
            if (!parse_dnsbl(tok, dc, *con)) return false;
        }
        else if (have == token_dnsbll) {
            if (!parse_dnsbll(tok, dc, *con)) return false;
        }
        else if (have == token_content) {
            if (!parse_content(tok, dc, *con)) return false;
        }
        else if (have == token_envto) {
            if (!parse_envto(tok, dc, *con)) return false;
        }
        else if (have == token_verify) {
            if (!parse_verify(tok, dc, *con)) return false;
        }
        else if (have == token_generic) {
            if (!parse_generic(tok, dc, *con)) return false;
        }
        else if (have == token_autowhite) {
            if (!parse_autowhite(tok, dc, *con)) return false;
        }
        else if (have == token_envfrom) {
            if (!parse_envfrom(tok, dc, *con)) return false;
        }
        else if (have == token_rate) {
            if (parent || dc.default_context) tok.token_error("rate limit ignored in non default context");
            if (!parse_rate(tok, dc, *con)) return false;
        }
        else if (have == token_context) {
            if (!parse_context(tok, dc, con)) return false;
        }
        else {
            tok.token_error("context keyword", have);
            return false;
        }
    }

    if (!tsa(tok, token_semi)) {
        delete con;
        return false;
    }
    dc.add_context(con);
    if (parent) parent->add_context(con);
    return true;
}


////////////////////////////////////////////////
// parse a config file
//
bool load_conf(CONFIG &dc, const char *fn) {
    int count = 0;
    TOKEN tok(fn, &dc.config_files);
    while (true) {
        const char *have = tok.next();
        if (!have) break;
        if (have == token_context) {
            if (!parse_context(tok, dc, NULL)) {
                tok.token_error("load_conf() failed to parse context");
                return false;
            }
            else count++;
        }
        else {
            tok.token_error(token_context, have);
            return false;
        }
    }
    tok.token_error("load_conf() found %d contexts in %s", count, fn);
    return (dc.default_context) ? true : false;
}


////////////////////////////////////////////////
// init the tokens
//
void token_init() {
    token_autowhite     = register_string("autowhite");
    token_black         = register_string("black");
    token_cctld         = register_string("cctld");
    token_content       = register_string("content");
    token_context       = register_string("context");
    token_dccbulk       = register_string("dcc_bulk_threshold");
    token_dccfrom       = register_string("dcc_from");
    token_dccgrey       = register_string("dcc_greylist");
    token_dccto         = register_string("dcc_to");
    token_default       = register_string("default");
    token_dnsbl         = register_string("dnsbl");
    token_dnsbll        = register_string("dnsbl_list");
    token_envfrom       = register_string("env_from");
    token_envto         = register_string("env_to");
    token_filter        = register_string("filter");
    token_generic       = register_string("generic");
    token_host_limit    = register_string("host_limit");
    token_html_limit    = register_string("html_limit");
    token_html_tags     = register_string("html_tags");
    token_ignore        = register_string("ignore");
    token_include       = register_string("include");
    token_inherit       = register_string("inherit");
    token_lbrace        = register_string("{");
    token_mailhost      = register_string("mail_host");
    token_many          = register_string("many");
    token_no            = register_string("no");
    token_off           = register_string("off");
    token_ok            = register_string("ok");
    token_ok2           = register_string("ok2");
    token_on            = register_string("on");
    token_rate          = register_string("rate_limit");
    token_rbrace        = register_string("}");
    token_require       = register_string("require_match");
    token_semi          = register_string(";");
    token_soft          = register_string("soft");
    token_spamassassin  = register_string("spamassassin");
    token_substitute    = register_string("substitute");
    token_tld           = register_string("tld");
    token_unknown       = register_string("unknown");
    token_uribl         = register_string("uribl");
    token_verify        = register_string("verify");
    token_white         = register_string("white");
    token_yes           = register_string("yes");

    if (gethostname(myhostname, HOST_NAME_MAX+1) != 0) {
        strncpy(myhostname, "localhost", HOST_NAME_MAX+1);
    }
    myhostname[HOST_NAME_MAX] = '\0'; // ensure null termination
    token_myhostname = register_string(myhostname);
}