Mercurial > dnsbl

annotate make-tld-conf.py @ 476:fcf66a7aead5 stable-6-0-78

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix dkim regex
author Carl Byington <carl@five-ten-sg.com>
date Sat, 20 Feb 2021 10:24:12 -0800
parents f5b394bec28c
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
467
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
1 #!/usr/bin/python3
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
2
276
19ff60eaab74 more tld entries
Carl Byington <carl@five-ten-sg.com>
parents: 270
diff changeset
3 import os, sys, time
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
4
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
5 def cmd(c):
467
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
6 print(c)
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
7 return os.popen(c).read().rstrip('\n')
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
8
291
9f0d9fcb58dd Never add auto-whitelist entries for outgoing mail from localhost
Carl Byington <carl@five-ten-sg.com>
parents: 289
diff changeset
9 url = 'https://publicsuffix.org/list/effective_tld_names.dat'
9f0d9fcb58dd Never add auto-whitelist entries for outgoing mail from localhost
Carl Byington <carl@five-ten-sg.com>
parents: 289
diff changeset
10 c = 'curl %s 2>/dev/null' % url
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
11 content = cmd(c)
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
12 f=open('tld.conf','w')
276
19ff60eaab74 more tld entries
Carl Byington <carl@five-ten-sg.com>
parents: 270
diff changeset
13 f.write(time.strftime("// generated by make-tld-conf.py on %Y-%m-%d %H:%M:%S\n\n"))
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
14 for x in content.split('\n'):
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
15 if (x == '') or (x[:2] == '//'): f.write('%s\n' % x)
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
16 else:
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
17 if x[:1] == '!':
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
18 x = x[1:]
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
19 prefix = '!'
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
20 else:
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
21 prefix = ''
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
22 yy = []
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
23 for xx in x.split('.'):
467
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
24 z = xx.encode('punycode').decode('utf8')
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
25 if z[-1:] == '-': yy.append(xx)
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
26 else: yy.append('xn--%s' % z)
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
27 y = '.'.join(yy)
289
6497944a9bd9 update tld.conf, ignore domains with no ns records
Carl Byington <carl@five-ten-sg.com>
parents: 276
diff changeset
28 if not (y == x):
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
29 f.write('// %s%s\n' % (prefix, x))
289
6497944a9bd9 update tld.conf, ignore domains with no ns records
Carl Byington <carl@five-ten-sg.com>
parents: 276
diff changeset
30 x = y
6497944a9bd9 update tld.conf, ignore domains with no ns records
Carl Byington <carl@five-ten-sg.com>
parents: 276
diff changeset
31 if not prefix and (len(x.split('.')) == 1):
300
3a3b0cbcaf6e update tld.conf for new zones
Carl Byington <carl@five-ten-sg.com>
parents: 291
diff changeset
32 rc = cmd("dig %s ns +short 2>&1" % x)
289
6497944a9bd9 update tld.conf, ignore domains with no ns records
Carl Byington <carl@five-ten-sg.com>
parents: 276
diff changeset
33 if not rc:
300
3a3b0cbcaf6e update tld.conf for new zones
Carl Byington <carl@five-ten-sg.com>
parents: 291
diff changeset
34 rc = cmd("sleep 3; dig %s ns +short 2>&1" % x)
289
6497944a9bd9 update tld.conf, ignore domains with no ns records
Carl Byington <carl@five-ten-sg.com>
parents: 276
diff changeset
35 if not rc: prefix = '// no ns record for '
6497944a9bd9 update tld.conf, ignore domains with no ns records
Carl Byington <carl@five-ten-sg.com>
parents: 276
diff changeset
36 f.write('%s%s\n' % (prefix, x))
307
c8144da09419 updated tld on 2015-10-15
Carl Byington <carl@five-ten-sg.com>
parents: 300
diff changeset
37 f.write('\n')
467
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
38 f.write('\n')
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
39 f.write('// add some TLDs from our rpz zone so names under them\n')
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
40 f.write('// can be added to uribl style lists\n')
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
41 f.write('\n')
445
78eedbbce636 round spamassassin scores; check >= rather than >
Carl Byington <carl@five-ten-sg.com>
parents: 307
diff changeset
42
467
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
43 c="grep '^\*.*multi.surbl' /etc/cron.daily/rpz | awk '{print $1}' | cut -c3- | sed -e 's/.multi.surbl.org//g' | grep '\.'"
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
44 content = cmd(c)
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
45 for x in content.split('\n'):
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
46 f.write(x + '\n')
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
47
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
48 f.write('\n')
f5b394bec28c allow checking names without A records on uribl lists; improve extraction of fake TLDs from our RPZ zone
Carl Byington <carl@five-ten-sg.com>
parents: 445
diff changeset
49 c="grep '^[^\*].*multi.surbl' /etc/cron.daily/rpz | egrep -v 'CNAME|g.write' | awk '{print $1}' | sed -e 's/.multi.surbl.org//g' | cut -d. -f2- | grep '\.'"
445
78eedbbce636 round spamassassin scores; check >= rather than >
Carl Byington <carl@five-ten-sg.com>
parents: 307
diff changeset
50 content = cmd(c)
78eedbbce636 round spamassassin scores; check >= rather than >
Carl Byington <carl@five-ten-sg.com>
parents: 307
diff changeset
51 for x in content.split('\n'):
78eedbbce636 round spamassassin scores; check >= rather than >
Carl Byington <carl@five-ten-sg.com>
parents: 307
diff changeset
52 f.write(x + '\n')
78eedbbce636 round spamassassin scores; check >= rather than >
Carl Byington <carl@five-ten-sg.com>
parents: 307
diff changeset
53
270
f92f24950bd3 Use mozilla prefix list for tld checking, Enable surbl/uribl/dbl rhs lists
Carl Byington <carl@five-ten-sg.com>
parents:
diff changeset
54 f.close()